michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

- Limit the depth of subdirectories to process in BulkImport. 

- Code polishing.
This commit is contained in:
Lampros Smyrnaios 2024-05-31 21:36:03 +03:00
parent edf064616a
commit 2241a89452
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java
View File

@ -449,7 +449,7 @@ public class BulkImportServiceImpl implements BulkImportService {
public List<String> getFileLocationsInsideDir(String directory) public List<String> getFileLocationsInsideDir(String directory)
{ {
List<String> fileLocations = null; List<String> fileLocations = null;
try ( Stream<Path> walkStream = Files.find(Paths.get(directory), Integer.MAX_VALUE, (filePath, fileAttr) -> fileAttr.isRegularFile()) ) try ( Stream<Path> walkStream = Files.find(Paths.get(directory), 20, (filePath, fileAttr) -> fileAttr.isRegularFile()) )
// In case we ever include other type-of-Files inside the same directory, we need to add this filter: "&& !filePath.toString().endsWith("name.ext")" // In case we ever include other type-of-Files inside the same directory, we need to add this filter: "&& !filePath.toString().endsWith("name.ext")"
{ {
fileLocations = walkStream.map(Path::toString).collect(Collectors.toList()); fileLocations = walkStream.map(Path::toString).collect(Collectors.toList());
@ -471,9 +471,8 @@ public class BulkImportServiceImpl implements BulkImportService {
md5 = DatatypeConverter.printHexBinary(md5MD.digest()).toLowerCase(); md5 = DatatypeConverter.printHexBinary(md5MD.digest()).toLowerCase();
} catch (Exception e) { } catch (Exception e) {
logger.error("Error when getting the MD5-hash for: " + string, e); logger.error("Error when getting the MD5-hash for: " + string, e);
return null;
} }
return md5; return md5; // It may be null.
} }
@ -487,7 +486,7 @@ public class BulkImportServiceImpl implements BulkImportService {
id = id.toLowerCase(); id = id.toLowerCase();
String idMd5Hash = getMD5Hash(id); String idMd5Hash = getMD5Hash(id);
if ( idMd5Hash == null ) if ( idMd5Hash == null ) // The error is logged inside.
return null; return null;
return (datasourcePrefix + "::" + idMd5Hash); return (datasourcePrefix + "::" + idMd5Hash);