D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Give a base path to FileSystem.get to resolve s3 file system

This commit is contained in:
Giambattista Bloisi 2024-10-21 16:42:19 +02:00
parent 4e97c96252
commit 1706e964f0
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkUpdateEntity.java
View File

@ -2,6 +2,7 @@
package eu.dnetlib.dhp.oa.dedup; package eu.dnetlib.dhp.oa.dedup;
import java.io.IOException; import java.io.IOException;
import java.net.URI;
import java.util.Map; import java.util.Map;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
@ -132,7 +133,7 @@ public class SparkUpdateEntity extends AbstractSparkAction {
boolean result = false; boolean result = false;
FileSystem fileSystem = FileSystem.get(new Configuration()); FileSystem fileSystem = FileSystem.get(URI.create(basePath), new Configuration());
FileStatus[] fileStatuses = fileSystem.listStatus(new Path(basePath)); FileStatus[] fileStatuses = fileSystem.listStatus(new Path(basePath));
for (FileStatus fs : fileStatuses) { for (FileStatus fs : fileStatuses) {