D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

dump #50

Merged
claudio.atzori merged 98 commits from miriam.baglioni/dnet-hadoop:dump into master 2020-11-04 18:07:01 +01:00
Conversation 1 Commits 98 Files Changed 51 +5 -4
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit b828587252 - Show all commits

9
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java
View File

@ -88,16 +88,16 @@ public class MakeTar implements Serializable {
write(fileSystem, inputPath, outputPath + ".tar", dir_name); write(fileSystem, inputPath, outputPath + ".tar", dir_name);
} else { } else {
int partNum = 0; int partNum = 0;
long remainingBytes = sourceSize % bytesPerSplit;
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
.listFiles( .listFiles(
new Path(inputPath), true); new Path(inputPath), true);
while (sourceSize > 0) { boolean next = fileStatusListIterator.hasNext();
TarArchiveOutputStream ar = getTar(fileSystem, outputPath + "_" + partNum + ".tar"); while (sourceSize > 0 && next) {
TarArchiveOutputStream ar = getTar(fileSystem, outputPath + "_" + (partNum + 1) + ".tar");
long current_size = 0; long current_size = 0;
while (fileStatusListIterator.hasNext() && current_size < bytesPerSplit) { while (next && current_size < bytesPerSplit) {
LocatedFileStatus fileStatus = fileStatusListIterator.next(); LocatedFileStatus fileStatus = fileStatusListIterator.next();
Path p = fileStatus.getPath(); Path p = fileStatus.getPath();
@ -125,6 +125,7 @@ public class MakeTar implements Serializable {
ar.closeArchiveEntry(); ar.closeArchiveEntry();
} }
next = fileStatusListIterator.hasNext();
} }
sourceSize = sourceSize - current_size; sourceSize = sourceSize - current_size;