dnet-hadoop/dhp-common/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java

173 lines
5.2 KiB
Java
Raw Normal View History

2020-11-04 18:12:43 +01:00
package eu.dnetlib.dhp.common;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.util.Optional;
2020-11-04 18:12:43 +01:00
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
2020-11-04 18:12:43 +01:00
import org.apache.hadoop.fs.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
2020-11-04 18:12:43 +01:00
public class MakeTarArchive implements Serializable {
private static final Logger log = LoggerFactory.getLogger(MakeTarArchive.class);
public static void main(String[] args) throws Exception {
String jsonConfiguration = IOUtils
.toString(
MakeTarArchive.class
.getResourceAsStream(
"/eu/dnetlib/dhp/common/input_maketar_parameters.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
parser.parseArgument(args);
final String outputPath = parser.get("hdfsPath");
log.info("hdfsPath: {}", outputPath);
final String hdfsNameNode = parser.get("nameNode");
log.info("nameNode: {}", hdfsNameNode);
final String inputPath = parser.get("sourcePath");
log.info("input path : {}", inputPath);
final int gBperSplit = Optional
.ofNullable(parser.get("splitSize"))
.map(Integer::valueOf)
.orElse(10);
Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fileSystem = FileSystem.get(conf);
makeTArArchive(fileSystem, inputPath, outputPath, gBperSplit);
}
public static void makeTArArchive(FileSystem fileSystem, String inputPath, String outputPath, int gBperSplit)
throws IOException {
RemoteIterator<LocatedFileStatus> dirIterator = fileSystem.listLocatedStatus(new Path(inputPath));
while (dirIterator.hasNext()) {
LocatedFileStatus fileStatus = dirIterator.next();
Path p = fileStatus.getPath();
String pathString = p.toString();
String entity = pathString.substring(pathString.lastIndexOf("/") + 1);
MakeTarArchive.tarMaxSize(fileSystem, pathString, outputPath + "/" + entity, entity, gBperSplit);
}
}
2020-11-04 18:12:43 +01:00
private static TarArchiveOutputStream getTar(FileSystem fileSystem, String outputPath) throws IOException {
Path hdfsWritePath = new Path(outputPath);
if (fileSystem.exists(hdfsWritePath)) {
fileSystem.delete(hdfsWritePath, true);
2020-11-04 18:12:43 +01:00
}
2021-08-11 12:13:22 +02:00
return new TarArchiveOutputStream(fileSystem.create(hdfsWritePath).getWrappedStream());
2020-11-04 18:12:43 +01:00
}
private static void write(FileSystem fileSystem, String inputPath, String outputPath, String dirName)
2020-11-04 18:12:43 +01:00
throws IOException {
2020-11-04 18:12:43 +01:00
Path hdfsWritePath = new Path(outputPath);
if (fileSystem.exists(hdfsWritePath)) {
fileSystem.delete(hdfsWritePath, true);
2020-11-04 18:12:43 +01:00
}
2021-08-11 12:13:22 +02:00
try (TarArchiveOutputStream ar = new TarArchiveOutputStream(
fileSystem.create(hdfsWritePath).getWrappedStream())) {
2021-08-11 12:13:22 +02:00
RemoteIterator<LocatedFileStatus> iterator = fileSystem
.listFiles(
new Path(inputPath), true);
2021-08-11 12:13:22 +02:00
while (iterator.hasNext()) {
writeCurrentFile(fileSystem, dirName, iterator, ar, 0);
2021-08-11 12:13:22 +02:00
}
2020-11-04 18:12:43 +01:00
}
}
2020-11-04 18:12:43 +01:00
public static void tarMaxSize(FileSystem fileSystem, String inputPath, String outputPath, String dir_name,
int gBperSplit) throws IOException {
final long bytesPerSplit = 1024L * 1024L * 1024L * gBperSplit;
2020-11-04 18:12:43 +01:00
long sourceSize = fileSystem.getContentSummary(new Path(inputPath)).getSpaceConsumed();
2020-11-04 18:12:43 +01:00
if (sourceSize < bytesPerSplit) {
write(fileSystem, inputPath, outputPath + ".tar", dir_name);
} else {
int partNum = 0;
2020-11-04 18:12:43 +01:00
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
.listFiles(
new Path(inputPath), true);
boolean next = fileStatusListIterator.hasNext();
while (next) {
try (TarArchiveOutputStream ar = getTar(fileSystem, outputPath + "_" + (partNum + 1) + ".tar")) {
long currentSize = 0;
while (next && currentSize < bytesPerSplit) {
currentSize = writeCurrentFile(fileSystem, dir_name, fileStatusListIterator, ar, currentSize);
next = fileStatusListIterator.hasNext();
}
partNum += 1;
}
2020-11-04 18:12:43 +01:00
}
}
}
private static long writeCurrentFile(FileSystem fileSystem, String dirName,
2020-11-04 18:12:43 +01:00
RemoteIterator<LocatedFileStatus> fileStatusListIterator,
TarArchiveOutputStream ar, long currentSize) throws IOException {
2020-11-04 18:12:43 +01:00
LocatedFileStatus fileStatus = fileStatusListIterator.next();
2020-11-04 18:12:43 +01:00
Path p = fileStatus.getPath();
String pString = p.toString();
if (!pString.endsWith("_SUCCESS")) {
String name = pString.substring(pString.lastIndexOf("/") + 1);
if (name.startsWith("part-") & name.length() > 10) {
String tmp = name.substring(0, 10);
if (name.contains(".")) {
tmp += name.substring(name.indexOf("."));
}
name = tmp;
}
TarArchiveEntry entry = new TarArchiveEntry(dirName + "/" + name);
2020-11-04 18:12:43 +01:00
entry.setSize(fileStatus.getLen());
currentSize += fileStatus.getLen();
2020-11-04 18:12:43 +01:00
ar.putArchiveEntry(entry);
2020-11-04 18:12:43 +01:00
InputStream is = fileSystem.open(fileStatus.getPath());
2020-11-04 18:12:43 +01:00
BufferedInputStream bis = new BufferedInputStream(is);
2020-11-04 18:12:43 +01:00
int count;
2021-05-14 10:58:12 +02:00
byte[] data = new byte[1024];
2020-11-04 18:12:43 +01:00
while ((count = bis.read(data, 0, data.length)) != -1) {
ar.write(data, 0, count);
}
bis.close();
ar.closeArchiveEntry();
2020-11-04 18:12:43 +01:00
}
return currentSize;
2020-11-04 18:12:43 +01:00
}
}