forked from D-Net/dnet-hadoop
Compare commits
1 Commits
| Author | SHA1 | Date |
|---|---|---|
 Miriam Baglioni
|
4a8ad34b72 |
|
|
@ -70,127 +70,6 @@ public class MakeTar implements Serializable {
|
|||
|
||||
}
|
||||
|
||||
// private static TarArchiveOutputStream getTar(FileSystem fileSystem, String outputPath) throws IOException {
|
||||
// Path hdfsWritePath = new Path(outputPath);
|
||||
// FSDataOutputStream fsDataOutputStream = null;
|
||||
// if (fileSystem.exists(hdfsWritePath)) {
|
||||
// fileSystem.delete(hdfsWritePath, true);
|
||||
//
|
||||
// }
|
||||
// fsDataOutputStream = fileSystem.create(hdfsWritePath);
|
||||
//
|
||||
// return new TarArchiveOutputStream(fsDataOutputStream.getWrappedStream());
|
||||
// }
|
||||
//
|
||||
// private static void writeMaxSize(FileSystem fileSystem, String inputPath, String outputPath, String dir_name,
|
||||
// int gBperSplit) throws IOException {
|
||||
// final long bytesPerSplit = 1024L * 1024L * 1024L * gBperSplit;
|
||||
//
|
||||
// long sourceSize = fileSystem.getContentSummary(new Path(inputPath)).getSpaceConsumed();
|
||||
//
|
||||
// final long numberOfSplits = sourceSize / bytesPerSplit;
|
||||
//
|
||||
// if (numberOfSplits < 2) {
|
||||
// write(fileSystem, inputPath, outputPath + ".tar", dir_name);
|
||||
// } else {
|
||||
// int partNum = 0;
|
||||
//
|
||||
// RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
|
||||
// .listFiles(
|
||||
// new Path(inputPath), true);
|
||||
// boolean next = fileStatusListIterator.hasNext();
|
||||
// while (sourceSize > 0 && next) {
|
||||
// TarArchiveOutputStream ar = getTar(fileSystem, outputPath + "_" + (partNum + 1) + ".tar");
|
||||
//
|
||||
// long current_size = 0;
|
||||
// while (next && current_size < bytesPerSplit) {
|
||||
// LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
||||
//
|
||||
// Path p = fileStatus.getPath();
|
||||
// String p_string = p.toString();
|
||||
// if (!p_string.endsWith("_SUCCESS")) {
|
||||
// String name = p_string.substring(p_string.lastIndexOf("/") + 1);
|
||||
// if (name.trim().equalsIgnoreCase("communities_infrastructures")) {
|
||||
// name = "communities_infrastructures.json";
|
||||
// }
|
||||
// TarArchiveEntry entry = new TarArchiveEntry(dir_name + "/" + name);
|
||||
// entry.setSize(fileStatus.getLen());
|
||||
// current_size += fileStatus.getLen();
|
||||
// ar.putArchiveEntry(entry);
|
||||
//
|
||||
// InputStream is = fileSystem.open(fileStatus.getPath());
|
||||
//
|
||||
// BufferedInputStream bis = new BufferedInputStream(is);
|
||||
//
|
||||
// int count;
|
||||
// byte data[] = new byte[1024];
|
||||
// while ((count = bis.read(data, 0, data.length)) != -1) {
|
||||
// ar.write(data, 0, count);
|
||||
// }
|
||||
// bis.close();
|
||||
// ar.closeArchiveEntry();
|
||||
//
|
||||
// }
|
||||
// next = fileStatusListIterator.hasNext();
|
||||
//
|
||||
// }
|
||||
// sourceSize = sourceSize - current_size;
|
||||
// partNum += 1;
|
||||
// ar.close();
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// private static void write(FileSystem fileSystem, String inputPath, String outputPath, String dir_name)
|
||||
// throws IOException {
|
||||
//
|
||||
// Path hdfsWritePath = new Path(outputPath);
|
||||
// FSDataOutputStream fsDataOutputStream = null;
|
||||
// if (fileSystem.exists(hdfsWritePath)) {
|
||||
// fileSystem.delete(hdfsWritePath, true);
|
||||
//
|
||||
// }
|
||||
// fsDataOutputStream = fileSystem.create(hdfsWritePath);
|
||||
//
|
||||
// TarArchiveOutputStream ar = new TarArchiveOutputStream(fsDataOutputStream.getWrappedStream());
|
||||
//
|
||||
// RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
|
||||
// .listFiles(
|
||||
// new Path(inputPath), true);
|
||||
//
|
||||
// while (fileStatusListIterator.hasNext()) {
|
||||
// LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
||||
//
|
||||
// Path p = fileStatus.getPath();
|
||||
// String p_string = p.toString();
|
||||
// if (!p_string.endsWith("_SUCCESS")) {
|
||||
// String name = p_string.substring(p_string.lastIndexOf("/") + 1);
|
||||
// if (name.trim().equalsIgnoreCase("communities_infrastructures")) {
|
||||
// name = "communities_infrastructures.json";
|
||||
// }
|
||||
// TarArchiveEntry entry = new TarArchiveEntry(dir_name + "/" + name);
|
||||
// entry.setSize(fileStatus.getLen());
|
||||
// ar.putArchiveEntry(entry);
|
||||
//
|
||||
// InputStream is = fileSystem.open(fileStatus.getPath());
|
||||
//
|
||||
// BufferedInputStream bis = new BufferedInputStream(is);
|
||||
//
|
||||
// int count;
|
||||
// byte data[] = new byte[1024];
|
||||
// while ((count = bis.read(data, 0, data.length)) != -1) {
|
||||
// ar.write(data, 0, count);
|
||||
// }
|
||||
// bis.close();
|
||||
// ar.closeArchiveEntry();
|
||||
//
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// ar.close();
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -156,7 +156,6 @@ public class ResultMapper implements Serializable {
|
|||
.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
|
||||
out.setContributor(contributorList);
|
||||
|
||||
// List<Country> countryList = new ArrayList<>();
|
||||
Optional
|
||||
.ofNullable(input.getCountry())
|
||||
.ifPresent(
|
||||
|
|
@ -188,7 +187,6 @@ public class ResultMapper implements Serializable {
|
|||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toList())));
|
||||
|
||||
// out.setCountry(countryList);
|
||||
|
||||
final List<String> coverageList = new ArrayList<>();
|
||||
Optional
|
||||
|
|
@ -286,7 +284,6 @@ public class ResultMapper implements Serializable {
|
|||
Optional
|
||||
.ofNullable(input.getSource())
|
||||
.ifPresent(value -> value.stream().forEach(s -> sourceList.add(s.getValue())));
|
||||
// out.setSource(input.getSource().stream().map(s -> s.getValue()).collect(Collectors.toList()));
|
||||
List<Subject> subjectList = new ArrayList<>();
|
||||
Optional
|
||||
.ofNullable(input.getSubject())
|
||||
|
|
@ -397,12 +394,7 @@ public class ResultMapper implements Serializable {
|
|||
|
||||
}
|
||||
|
||||
private static <I extends Instance> void setCommonValue(eu.dnetlib.dhp.schema.oaf.Instance i, I instance) {// <I
|
||||
// extends
|
||||
// Instance>
|
||||
// I
|
||||
// getInstance(eu.dnetlib.dhp.schema.oaf.Instance
|
||||
// i, boolean graph) {
|
||||
private static <I extends Instance> void setCommonValue(eu.dnetlib.dhp.schema.oaf.Instance i, I instance) {
|
||||
Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
|
||||
.ofNullable(i.getAccessright());
|
||||
if (opAr.isPresent()) {
|
||||
|
|
@ -427,7 +419,7 @@ public class ResultMapper implements Serializable {
|
|||
Optional
|
||||
.ofNullable(i.getRefereed())
|
||||
.ifPresent(value -> instance.setRefereed(value.getClassname()));
|
||||
// .ifPresent(value -> instance.setRefereed(value.getValue()));
|
||||
|
||||
Optional
|
||||
.ofNullable(i.getInstancetype())
|
||||
.ifPresent(value -> instance.setType(value.getClassname()));
|
||||
|
|
@ -437,10 +429,10 @@ public class ResultMapper implements Serializable {
|
|||
|
||||
private static List<Provenance> getUniqueProvenance(List<Provenance> provenance) {
|
||||
Provenance iProv = new Provenance();
|
||||
// iProv.setProvenance(Constants.INFERRED);
|
||||
|
||||
|
||||
Provenance hProv = new Provenance();
|
||||
// hProv.setProvenance(Constants.HARVESTED);
|
||||
|
||||
Provenance lProv = new Provenance();
|
||||
|
||||
for (Provenance p : provenance) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue