forked from D-Net/dnet-hadoop
Compare commits
1 Commits
Author | SHA1 | Date |
---|---|---|
Miriam Baglioni | 4a8ad34b72 |
|
@ -70,127 +70,6 @@ public class MakeTar implements Serializable {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// private static TarArchiveOutputStream getTar(FileSystem fileSystem, String outputPath) throws IOException {
|
|
||||||
// Path hdfsWritePath = new Path(outputPath);
|
|
||||||
// FSDataOutputStream fsDataOutputStream = null;
|
|
||||||
// if (fileSystem.exists(hdfsWritePath)) {
|
|
||||||
// fileSystem.delete(hdfsWritePath, true);
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
// fsDataOutputStream = fileSystem.create(hdfsWritePath);
|
|
||||||
//
|
|
||||||
// return new TarArchiveOutputStream(fsDataOutputStream.getWrappedStream());
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private static void writeMaxSize(FileSystem fileSystem, String inputPath, String outputPath, String dir_name,
|
|
||||||
// int gBperSplit) throws IOException {
|
|
||||||
// final long bytesPerSplit = 1024L * 1024L * 1024L * gBperSplit;
|
|
||||||
//
|
|
||||||
// long sourceSize = fileSystem.getContentSummary(new Path(inputPath)).getSpaceConsumed();
|
|
||||||
//
|
|
||||||
// final long numberOfSplits = sourceSize / bytesPerSplit;
|
|
||||||
//
|
|
||||||
// if (numberOfSplits < 2) {
|
|
||||||
// write(fileSystem, inputPath, outputPath + ".tar", dir_name);
|
|
||||||
// } else {
|
|
||||||
// int partNum = 0;
|
|
||||||
//
|
|
||||||
// RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
|
|
||||||
// .listFiles(
|
|
||||||
// new Path(inputPath), true);
|
|
||||||
// boolean next = fileStatusListIterator.hasNext();
|
|
||||||
// while (sourceSize > 0 && next) {
|
|
||||||
// TarArchiveOutputStream ar = getTar(fileSystem, outputPath + "_" + (partNum + 1) + ".tar");
|
|
||||||
//
|
|
||||||
// long current_size = 0;
|
|
||||||
// while (next && current_size < bytesPerSplit) {
|
|
||||||
// LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
|
||||||
//
|
|
||||||
// Path p = fileStatus.getPath();
|
|
||||||
// String p_string = p.toString();
|
|
||||||
// if (!p_string.endsWith("_SUCCESS")) {
|
|
||||||
// String name = p_string.substring(p_string.lastIndexOf("/") + 1);
|
|
||||||
// if (name.trim().equalsIgnoreCase("communities_infrastructures")) {
|
|
||||||
// name = "communities_infrastructures.json";
|
|
||||||
// }
|
|
||||||
// TarArchiveEntry entry = new TarArchiveEntry(dir_name + "/" + name);
|
|
||||||
// entry.setSize(fileStatus.getLen());
|
|
||||||
// current_size += fileStatus.getLen();
|
|
||||||
// ar.putArchiveEntry(entry);
|
|
||||||
//
|
|
||||||
// InputStream is = fileSystem.open(fileStatus.getPath());
|
|
||||||
//
|
|
||||||
// BufferedInputStream bis = new BufferedInputStream(is);
|
|
||||||
//
|
|
||||||
// int count;
|
|
||||||
// byte data[] = new byte[1024];
|
|
||||||
// while ((count = bis.read(data, 0, data.length)) != -1) {
|
|
||||||
// ar.write(data, 0, count);
|
|
||||||
// }
|
|
||||||
// bis.close();
|
|
||||||
// ar.closeArchiveEntry();
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
// next = fileStatusListIterator.hasNext();
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
// sourceSize = sourceSize - current_size;
|
|
||||||
// partNum += 1;
|
|
||||||
// ar.close();
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private static void write(FileSystem fileSystem, String inputPath, String outputPath, String dir_name)
|
|
||||||
// throws IOException {
|
|
||||||
//
|
|
||||||
// Path hdfsWritePath = new Path(outputPath);
|
|
||||||
// FSDataOutputStream fsDataOutputStream = null;
|
|
||||||
// if (fileSystem.exists(hdfsWritePath)) {
|
|
||||||
// fileSystem.delete(hdfsWritePath, true);
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
// fsDataOutputStream = fileSystem.create(hdfsWritePath);
|
|
||||||
//
|
|
||||||
// TarArchiveOutputStream ar = new TarArchiveOutputStream(fsDataOutputStream.getWrappedStream());
|
|
||||||
//
|
|
||||||
// RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
|
|
||||||
// .listFiles(
|
|
||||||
// new Path(inputPath), true);
|
|
||||||
//
|
|
||||||
// while (fileStatusListIterator.hasNext()) {
|
|
||||||
// LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
|
||||||
//
|
|
||||||
// Path p = fileStatus.getPath();
|
|
||||||
// String p_string = p.toString();
|
|
||||||
// if (!p_string.endsWith("_SUCCESS")) {
|
|
||||||
// String name = p_string.substring(p_string.lastIndexOf("/") + 1);
|
|
||||||
// if (name.trim().equalsIgnoreCase("communities_infrastructures")) {
|
|
||||||
// name = "communities_infrastructures.json";
|
|
||||||
// }
|
|
||||||
// TarArchiveEntry entry = new TarArchiveEntry(dir_name + "/" + name);
|
|
||||||
// entry.setSize(fileStatus.getLen());
|
|
||||||
// ar.putArchiveEntry(entry);
|
|
||||||
//
|
|
||||||
// InputStream is = fileSystem.open(fileStatus.getPath());
|
|
||||||
//
|
|
||||||
// BufferedInputStream bis = new BufferedInputStream(is);
|
|
||||||
//
|
|
||||||
// int count;
|
|
||||||
// byte data[] = new byte[1024];
|
|
||||||
// while ((count = bis.read(data, 0, data.length)) != -1) {
|
|
||||||
// ar.write(data, 0, count);
|
|
||||||
// }
|
|
||||||
// bis.close();
|
|
||||||
// ar.closeArchiveEntry();
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// ar.close();
|
|
||||||
// }
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -156,7 +156,6 @@ public class ResultMapper implements Serializable {
|
||||||
.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
|
.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
|
||||||
out.setContributor(contributorList);
|
out.setContributor(contributorList);
|
||||||
|
|
||||||
// List<Country> countryList = new ArrayList<>();
|
|
||||||
Optional
|
Optional
|
||||||
.ofNullable(input.getCountry())
|
.ofNullable(input.getCountry())
|
||||||
.ifPresent(
|
.ifPresent(
|
||||||
|
@ -188,7 +187,6 @@ public class ResultMapper implements Serializable {
|
||||||
.filter(Objects::nonNull)
|
.filter(Objects::nonNull)
|
||||||
.collect(Collectors.toList())));
|
.collect(Collectors.toList())));
|
||||||
|
|
||||||
// out.setCountry(countryList);
|
|
||||||
|
|
||||||
final List<String> coverageList = new ArrayList<>();
|
final List<String> coverageList = new ArrayList<>();
|
||||||
Optional
|
Optional
|
||||||
|
@ -286,7 +284,6 @@ public class ResultMapper implements Serializable {
|
||||||
Optional
|
Optional
|
||||||
.ofNullable(input.getSource())
|
.ofNullable(input.getSource())
|
||||||
.ifPresent(value -> value.stream().forEach(s -> sourceList.add(s.getValue())));
|
.ifPresent(value -> value.stream().forEach(s -> sourceList.add(s.getValue())));
|
||||||
// out.setSource(input.getSource().stream().map(s -> s.getValue()).collect(Collectors.toList()));
|
|
||||||
List<Subject> subjectList = new ArrayList<>();
|
List<Subject> subjectList = new ArrayList<>();
|
||||||
Optional
|
Optional
|
||||||
.ofNullable(input.getSubject())
|
.ofNullable(input.getSubject())
|
||||||
|
@ -397,12 +394,7 @@ public class ResultMapper implements Serializable {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static <I extends Instance> void setCommonValue(eu.dnetlib.dhp.schema.oaf.Instance i, I instance) {// <I
|
private static <I extends Instance> void setCommonValue(eu.dnetlib.dhp.schema.oaf.Instance i, I instance) {
|
||||||
// extends
|
|
||||||
// Instance>
|
|
||||||
// I
|
|
||||||
// getInstance(eu.dnetlib.dhp.schema.oaf.Instance
|
|
||||||
// i, boolean graph) {
|
|
||||||
Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
|
Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
|
||||||
.ofNullable(i.getAccessright());
|
.ofNullable(i.getAccessright());
|
||||||
if (opAr.isPresent()) {
|
if (opAr.isPresent()) {
|
||||||
|
@ -427,7 +419,7 @@ public class ResultMapper implements Serializable {
|
||||||
Optional
|
Optional
|
||||||
.ofNullable(i.getRefereed())
|
.ofNullable(i.getRefereed())
|
||||||
.ifPresent(value -> instance.setRefereed(value.getClassname()));
|
.ifPresent(value -> instance.setRefereed(value.getClassname()));
|
||||||
// .ifPresent(value -> instance.setRefereed(value.getValue()));
|
|
||||||
Optional
|
Optional
|
||||||
.ofNullable(i.getInstancetype())
|
.ofNullable(i.getInstancetype())
|
||||||
.ifPresent(value -> instance.setType(value.getClassname()));
|
.ifPresent(value -> instance.setType(value.getClassname()));
|
||||||
|
@ -437,10 +429,10 @@ public class ResultMapper implements Serializable {
|
||||||
|
|
||||||
private static List<Provenance> getUniqueProvenance(List<Provenance> provenance) {
|
private static List<Provenance> getUniqueProvenance(List<Provenance> provenance) {
|
||||||
Provenance iProv = new Provenance();
|
Provenance iProv = new Provenance();
|
||||||
// iProv.setProvenance(Constants.INFERRED);
|
|
||||||
|
|
||||||
Provenance hProv = new Provenance();
|
Provenance hProv = new Provenance();
|
||||||
// hProv.setProvenance(Constants.HARVESTED);
|
|
||||||
Provenance lProv = new Provenance();
|
Provenance lProv = new Provenance();
|
||||||
|
|
||||||
for (Provenance p : provenance) {
|
for (Provenance p : provenance) {
|
||||||
|
|
Loading…
Reference in New Issue