michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

- Handle the case when the "webHDFSBaseUrl" does not use HTTPS. 

- Improve error-reporting when uploading a file to HDFS.
This commit is contained in:
Lampros Smyrnaios 2023-10-19 11:59:37 +03:00
parent 40729c6295
commit df0ea62a5a
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java
View File

@ -448,7 +448,9 @@ public class ParquetFileUtils {
}
//logger.trace("The target location is: " + location + "\nWill do a silent redirect to HTTPS."); // DEBUG!
location = StringUtils.replace(location, "http:", "https:", 1);
// In case the WebHDFS uses https, then perform the offline redirect to https here (to avoid the live redirection.)
if ( webHDFSBaseUrl.startsWith("https:") )
location = StringUtils.replace(location, "http:", "https:", 1);
// Unless we handle this here, we have to either complicate the process by handling the https-redirect or in any-way getting a hit in performance by having one more step each time ww want to upload a file.
conn = (HttpURLConnection) (new URL(location)).openConnection(); // This already contains the "user.name" parameter.
@ -487,8 +489,8 @@ public class ParquetFileUtils {
// Using the "load data inpath" command, he files are MOVED, not copied! So we don't have to delete them afterwards.
// See: https://docs.cloudera.com/documentation/enterprise/latest/topics/impala_load_data.html
} catch (Throwable e) {
String errorMsg = "Error while uploading parquet file \"" + parquetFileFullLocalPath + "\" to HDFS!\n" + e;
logger.error(errorMsg);
String errorMsg = "Error while uploading parquet file \"" + parquetFileFullLocalPath + "\" to HDFS!\n" + e.getMessage();
logger.error(errorMsg, e);
return errorMsg;
}