michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

- Improve the "getDataForPayloadPrefillQuery". 

- Improve some error-messages.
This commit is contained in:
Lampros Smyrnaios 2023-10-21 11:31:31 +03:00
parent 44c2fe7418
commit a7581335f1
2 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -768,10 +768,11 @@ public class FileUtils {
relatedIDsStringBuilder.append(")");
// Get the id and url of any
String getDataForPayloadPrefillQuery = "select pu.id, pu.url\n" +
String getDataForPayloadPrefillQuery = "select distinct pu.id, pu.url\n" +
"from " + DatabaseConnector.databaseName + ".publication_urls pu\n" +
"left anti join " + DatabaseConnector.databaseName + ".attempt a on a.id=pu.id and a.original_url=pu.url\n" +
"left anti join " + DatabaseConnector.databaseName + ".payload p on p.id=pu.id and p.original_url=pu.url\n" +
"left anti join " + DatabaseConnector.databaseName + ".assignment asgn on asgn.id=pu.id and asgn.original_url=pu.url\n" +
"where pu.url in " + relatedIDsStringBuilder;
//logger.trace("getDataForPayloadPrefillQuery:\n" + getDataForPayloadPrefillQuery);

4
src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java
View File

@ -489,9 +489,9 @@ public class ParquetFileUtils {
// Using the "load data inpath" command, he files are MOVED, not copied! So we don't have to delete them afterwards.
// See: https://docs.cloudera.com/documentation/enterprise/latest/topics/impala_load_data.html
} catch (Throwable e) {
String errorMsg = "Error while uploading parquet file \"" + parquetFileFullLocalPath + "\" to HDFS!\n" + e.getMessage();
String errorMsg = "Error while uploading parquet file \"" + parquetFileFullLocalPath + "\" to HDFS!\n";
logger.error(errorMsg, e);
return errorMsg;
return errorMsg + e.getMessage();
}
// The local parquet file will be deleted later.