- Show a warning, if the "numOfUnretrievedFiles" is over 50.
- Delete gradle .zip file after installation. - Code polishing.
This commit is contained in:
parent
088cf73b30
commit
dc97b323c9
|
@ -66,6 +66,7 @@ if [[ justRun -eq 0 ]]; then
|
||||||
echo -e "\nAsking for sudo, in order to install 'gradle'..\n"
|
echo -e "\nAsking for sudo, in order to install 'gradle'..\n"
|
||||||
sudo mkdir /opt/gradle
|
sudo mkdir /opt/gradle
|
||||||
sudo apt install -y unzip && sudo unzip -d /opt/gradle gradle-${gradleVersion}-bin.zip
|
sudo apt install -y unzip && sudo unzip -d /opt/gradle gradle-${gradleVersion}-bin.zip
|
||||||
|
sudo rm -rf gradle-${gradleVersion}-bin.zip
|
||||||
#ls /opt/gradle/gradle-${gradleVersion} # For debugging installation
|
#ls /opt/gradle/gradle-${gradleVersion} # For debugging installation
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
@ -185,6 +185,7 @@ public class PublicationsRetrieverPlugin {
|
||||||
for ( Assignment assignment : assignments )
|
for ( Assignment assignment : assignments )
|
||||||
urlIdsWithDatasourceIds.put(assignment.getId(), assignment.getDatasource().getId());
|
urlIdsWithDatasourceIds.put(assignment.getId(), assignment.getDatasource().getId());
|
||||||
|
|
||||||
|
int numOfUnretrievedFiles = 0;
|
||||||
Timestamp timestamp = new Timestamp(System.currentTimeMillis()); // Store it here, in order to have the same for all current records.
|
Timestamp timestamp = new Timestamp(System.currentTimeMillis()); // Store it here, in order to have the same for all current records.
|
||||||
|
|
||||||
for ( DataToBeLogged data : FileUtils.dataToBeLoggedList )
|
for ( DataToBeLogged data : FileUtils.dataToBeLoggedList )
|
||||||
|
@ -247,11 +248,13 @@ public class PublicationsRetrieverPlugin {
|
||||||
else if ( ! comment.startsWith(HttpConnUtils.docFileNotRetrievedMessage, 0) ) { // If it was downloaded without an error.
|
else if ( ! comment.startsWith(HttpConnUtils.docFileNotRetrievedMessage, 0) ) { // If it was downloaded without an error.
|
||||||
fileLocation = comment; // This is the full-file-path.
|
fileLocation = comment; // This is the full-file-path.
|
||||||
mimeType = "application/pdf";
|
mimeType = "application/pdf";
|
||||||
} else // Else the file was not retrieved, so all file-related data are kept "null".
|
} else { // Else the file was not retrieved, so all file-related data are kept "null".
|
||||||
|
numOfUnretrievedFiles ++;
|
||||||
error = new Error(Error.ErrorType.couldRetry, comment); // We can still try to download it from the found docUrl, in the future.
|
error = new Error(Error.ErrorType.couldRetry, comment); // We can still try to download it from the found docUrl, in the future.
|
||||||
|
}
|
||||||
|
|
||||||
if ( error == null ) // If the file was retrieved, in any time.
|
if ( error == null ) // If the file was retrieved, in any time.
|
||||||
error = new Error(Error.ErrorType.couldRetry, null); // We do not want to send a "null" Error-object, since it just adds more complicated handling in the controller..
|
error = new Error(Error.ErrorType.couldRetry, null); // We do not want to send a "null" Error-object, since it just adds more complicated handling in the Controller..
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
status = UrlReport.StatusType.non_accessible;
|
status = UrlReport.StatusType.non_accessible;
|
||||||
|
@ -281,6 +284,9 @@ public class PublicationsRetrieverPlugin {
|
||||||
AssignmentsHandler.urlReports.add(new UrlReport(status, payload, error));
|
AssignmentsHandler.urlReports.add(new UrlReport(status, payload, error));
|
||||||
}// end-for
|
}// end-for
|
||||||
FileUtils.dataToBeLoggedList.clear(); // Empty the list, to be re-populated by the next batch / assignment.
|
FileUtils.dataToBeLoggedList.clear(); // Empty the list, to be re-populated by the next batch / assignment.
|
||||||
|
|
||||||
|
if ( numOfUnretrievedFiles > 50 )
|
||||||
|
logger.warn("The number of non-retrieved files is: " + numOfUnretrievedFiles);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,7 +38,7 @@ public class FullTextsController {
|
||||||
public Object getFullTexts(@PathVariable long assignmentsCounter, @PathVariable int totalBatches, @PathVariable int batchCounter, @PathVariable List<String> fileNamesWithExtensions)
|
public Object getFullTexts(@PathVariable long assignmentsCounter, @PathVariable int totalBatches, @PathVariable int batchCounter, @PathVariable List<String> fileNamesWithExtensions)
|
||||||
{
|
{
|
||||||
int fileNamesListNum = fileNamesWithExtensions.size();
|
int fileNamesListNum = fileNamesWithExtensions.size();
|
||||||
if ( (fileNamesListNum == 1) && (fileNamesWithExtensions.get(0).length() == 0) ) { // In case the last "/" in the url was given (without any files following), then this list will not be empty, but have one empty item instead.
|
if ( (fileNamesListNum == 1) && (fileNamesWithExtensions.get(0).isEmpty()) ) { // In case the last "/" in the url was given (without any files following), then this list will not be empty, but have one empty item instead.
|
||||||
// In case the url does not end in "/", then Spring will automatically return an "HTTP-BadRequest".
|
// In case the url does not end in "/", then Spring will automatically return an "HTTP-BadRequest".
|
||||||
String errorMsg = "An empty \"fileNamesWithExtensions\" list was given from assignments_" + assignmentsCounter + ", for batch_" + batchCounter;
|
String errorMsg = "An empty \"fileNamesWithExtensions\" list was given from assignments_" + assignmentsCounter + ", for batch_" + batchCounter;
|
||||||
logger.error(errorMsg);
|
logger.error(errorMsg);
|
||||||
|
@ -91,8 +91,9 @@ public class FullTextsController {
|
||||||
} finally {
|
} finally {
|
||||||
// The ".tar.zstd" file of this batch, for which we pass a steam to the Controller, will be deleted by the next batch or in the end of these assignments.
|
// The ".tar.zstd" file of this batch, for which we pass a steam to the Controller, will be deleted by the next batch or in the end of these assignments.
|
||||||
// Now we will delete the zstd file of the previous assignments.
|
// Now we will delete the zstd file of the previous assignments.
|
||||||
if ( batchCounter >= 2 )
|
int previousBatchCounter = (batchCounter -1);
|
||||||
deleteFile(currentAssignmentsBaseFullTextsPath + "assignments_" + assignmentsCounter + "_full-texts_" + (batchCounter -1) + ".tar.zstd");
|
if ( previousBatchCounter >= 1 )
|
||||||
|
deleteFile(currentAssignmentsBaseFullTextsPath + "assignments_" + assignmentsCounter + "_full-texts_" + previousBatchCounter + ".tar.zstd");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue