- The worker will store the files in its local file-system and will send them to the controller in batches, after the latter requests them. When all files from a given assignments-num are sent, the files will be deleted from the Worker, in a scheduled-job.
- Implement the "getFullTexts"-endpoint, which returns the requested full-texts in a zip file. - Implement the "getFullText"-endpoint, which returns the requested full-text. - Implement the "getHandledAssignmentsCounts"-endpoint which returns the assignments-numbers, which were handled by that worker. - Make sure each urlReport has the same "Date" for a given assignments-number. Also, make sure the "size" and "hash" have a "null" value, in case the full-text was not found. - Check and log thread-pool shutdown errors. - Add the stack-trace in the error-logs, instead of the Stderr. - Update SpringBoot dependency. - Change log levels. - Code cleanup.master
parent
3220c97373
commit
20b71164d5
@ -0,0 +1,107 @@
|
||||
package eu.openaire.urls_worker.controllers;
|
||||
|
||||
import eu.openaire.urls_worker.services.FileStorageService;
|
||||
import eu.openaire.urls_worker.util.FilesZipper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("full-texts/")
|
||||
public class FullTextsController {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(GeneralController.class);
|
||||
|
||||
private final FileStorageService fileStorageService;
|
||||
|
||||
public static HashMap<Long, Boolean> assignmentsNumsHandledAndLocallyDeleted = new HashMap<>();
|
||||
|
||||
public static String assignmentsBaseDir = null;
|
||||
|
||||
|
||||
public FullTextsController(FileStorageService fileStorageService) {
|
||||
this.fileStorageService = fileStorageService;
|
||||
assignmentsBaseDir = FileStorageService.assignmentsLocation.toString() + File.separator;
|
||||
}
|
||||
|
||||
|
||||
@GetMapping("getFullTexts/{assignmentsCounter:[\\d]+}/{totalZipBatches:[\\d]+}/{zipBatchCounter:[\\d]+}/{fileNamesWithExtensions}")
|
||||
public ResponseEntity<?> getMultipleFullTexts(@PathVariable long assignmentsCounter, @PathVariable int totalZipBatches, @PathVariable int zipBatchCounter, @PathVariable List<String> fileNamesWithExtensions, HttpServletRequest request) {
|
||||
|
||||
logger.info("Received a \"getMultipleFullTexts\" request for returning a zip-file containing " + fileNamesWithExtensions.size() + " full-texts, from assignments-" + assignmentsCounter + ", for batch-" + zipBatchCounter);
|
||||
|
||||
String currentAssignmentsBaseFullTextsPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator;
|
||||
|
||||
File zipFile = FilesZipper.zipMultipleFilesAndGetZip(assignmentsCounter, zipBatchCounter, fileNamesWithExtensions, currentAssignmentsBaseFullTextsPath);
|
||||
if ( zipFile == null ) {
|
||||
String errorMsg = "Failed to create the zip file for \"zipBatchCounter\"-" + zipBatchCounter;
|
||||
logger.error(errorMsg);
|
||||
return ResponseEntity.internalServerError().body(errorMsg);
|
||||
}
|
||||
|
||||
String zipName = zipFile.getName();
|
||||
String zipFileFullPath = currentAssignmentsBaseFullTextsPath + zipName;
|
||||
ByteArrayOutputStream byteArrayOutputStream = this.fileStorageService.loadFileAsAStream(zipFileFullPath);
|
||||
if ( byteArrayOutputStream == null )
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body("Could not load file: " + zipFileFullPath);
|
||||
|
||||
// If this is the last batch for this assignments-count, then make sure it is deleted in the next scheduled delete-operation.
|
||||
if ( zipBatchCounter == totalZipBatches ) {
|
||||
assignmentsNumsHandledAndLocallyDeleted.put(assignmentsCounter, false);
|
||||
logger.debug("Will return the last batch of Assignments_" + assignmentsCounter + " to the Controller and these assignments will be deleted later.");
|
||||
}
|
||||
|
||||
String contentType = request.getServletContext().getMimeType(zipFileFullPath);
|
||||
if ( contentType == null )
|
||||
contentType = "application/octet-stream";
|
||||
|
||||
logger.info("Sending the zip file \"" + zipFileFullPath + "\".");
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.parseMediaType(contentType))
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + zipName + "\"")
|
||||
.body(byteArrayOutputStream.toByteArray());
|
||||
}
|
||||
|
||||
|
||||
@GetMapping("getFullText/{assignmentsCounter:[\\d]+]}/{fileNameWithExtension:[\\w]+.[\\w]{2,10}}")
|
||||
public ResponseEntity<?> getFullText(@PathVariable long assignmentsCounter, @PathVariable String fileNameWithExtension, HttpServletRequest request) {
|
||||
|
||||
logger.info("Received a \"getFullText\" request.");
|
||||
String fullTextFile = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension;
|
||||
File file = new File(fullTextFile);
|
||||
if ( !file.isFile() ) {
|
||||
logger.error("The file \"" + fullTextFile + "\" does not exist!");
|
||||
return ResponseEntity.notFound().build();
|
||||
}
|
||||
|
||||
Resource resource = this.fileStorageService.loadFileAsResource(fullTextFile);
|
||||
if ( resource == null )
|
||||
return ResponseEntity.internalServerError().body("Could not load file: " + fullTextFile);
|
||||
|
||||
String contentType = null;
|
||||
contentType = request.getServletContext().getMimeType(file.getAbsolutePath());
|
||||
if ( contentType == null ) {
|
||||
contentType = "application/octet-stream";
|
||||
}
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.parseMediaType(contentType))
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + resource.getFilename() + "\"")
|
||||
.body(resource);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package eu.openaire.urls_worker.exceptions;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
|
||||
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
|
||||
public class FileStorageException extends RuntimeException {
|
||||
|
||||
public FileStorageException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public FileStorageException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,85 @@
|
||||
package eu.openaire.urls_worker.services;
|
||||
|
||||
import eu.openaire.urls_worker.exceptions.FileStorageException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.core.io.UrlResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.*;
|
||||
import java.util.Properties;
|
||||
|
||||
|
||||
@Service
|
||||
public class FileStorageService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(FileStorageService.class);
|
||||
|
||||
public static Path assignmentsLocation = null;
|
||||
|
||||
static {
|
||||
String springPropertiesFile = System.getProperty("user.dir") + File.separator + "src" + File.separator + "main" + File.separator + "resources" + File.separator + "application.properties";
|
||||
FileReader fReader = null;
|
||||
try {
|
||||
fReader = new FileReader(springPropertiesFile);
|
||||
Properties props = new Properties();
|
||||
props.load(fReader); // Load jdbc related properties.
|
||||
String assignmentsDir = props.getProperty("file.assignments-dir");
|
||||
assignmentsLocation = Paths.get(assignmentsDir).toAbsolutePath().normalize();
|
||||
} catch (java.io.FileNotFoundException fnfe) {
|
||||
logger.error("The properties file was not found!", fnfe);
|
||||
System.exit(-10);
|
||||
} catch (IOException ioe) {
|
||||
logger.error("I/O error when reading the properties file!", ioe);
|
||||
System.exit(-10);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
public FileStorageService() throws FileStorageException {
|
||||
try {
|
||||
Files.createDirectories(assignmentsLocation);
|
||||
} catch (Exception ex) {
|
||||
throw new FileStorageException("Could not create the directory where the uploaded files will be stored.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static final int bufferSize = 20971520;
|
||||
public ByteArrayOutputStream loadFileAsAStream(String fullFileName)
|
||||
{
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(bufferSize); // 20 MB
|
||||
try {
|
||||
FileInputStream fileInputStream = new FileInputStream(fullFileName);
|
||||
int bytesRead;
|
||||
byte[] byteBuffer = new byte[bufferSize];
|
||||
while ( (bytesRead = fileInputStream.read(byteBuffer, 0, bufferSize)) > 0 )
|
||||
byteArrayOutputStream.write(byteBuffer, 0, bytesRead);
|
||||
|
||||
return byteArrayOutputStream;
|
||||
} catch (Exception e) {
|
||||
if ( e instanceof FileNotFoundException )
|
||||
logger.error("File \"" + fullFileName + "\" is not a file!");
|
||||
else
|
||||
logger.error(e.getMessage(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public Resource loadFileAsResource(String fullFileName) {
|
||||
try {
|
||||
Path filePath = assignmentsLocation.resolve(fullFileName).normalize();
|
||||
Resource resource = new UrlResource(filePath.toUri());
|
||||
return resource.exists() ? resource : null;
|
||||
} catch (Exception e) {
|
||||
logger.error("Error when loading file: " + fullFileName, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,80 @@
|
||||
package eu.openaire.urls_worker.util;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.List;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
|
||||
public class FilesZipper
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(FilesZipper.class);
|
||||
|
||||
|
||||
public static File zipMultipleFilesAndGetZip(long assignmentsCounter, int zipBatchCounter, List<String> filesToZip, String baseDirectory)
|
||||
{
|
||||
File zipFile = null;
|
||||
ZipOutputStream zos = null;
|
||||
try {
|
||||
String zipFilename = baseDirectory + "assignments_" + assignmentsCounter + "_full-texts_" + zipBatchCounter + ".zip";
|
||||
// For example: assignments_2_full-texts_4.zip | where < 4 > is referred to the 4th batch of files requested by the controller.
|
||||
zipFile = new File(zipFilename);
|
||||
zos = new ZipOutputStream(new FileOutputStream(zipFile));
|
||||
|
||||
// Iterate over the given full-texts and add them to the zip.
|
||||
for ( String file : filesToZip )
|
||||
{
|
||||
zipAFile(file, zos, baseDirectory);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("", e);
|
||||
return null;
|
||||
} finally {
|
||||
try {
|
||||
if ( zos != null )
|
||||
zos.close();
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return zipFile;
|
||||
}
|
||||
|
||||
|
||||
private static boolean zipAFile(String fileName, ZipOutputStream zos, String baseDir)
|
||||
{
|
||||
final int BUFFER = 1048576; // 1 MB
|
||||
byte[] data = new byte[BUFFER];
|
||||
BufferedInputStream bis = null;
|
||||
String fullFileName = baseDir + fileName;
|
||||
try {
|
||||
FileInputStream fis = new FileInputStream(fullFileName);
|
||||
bis = new BufferedInputStream(fis, BUFFER);
|
||||
zos.putNextEntry(new ZipEntry(fileName));
|
||||
int count;
|
||||
while ( (count = bis.read(data, 0, BUFFER)) != -1 ) {
|
||||
zos.write(data, 0, count);
|
||||
}
|
||||
zos.closeEntry(); // close the entry here (not the ZipOutputStream)
|
||||
} catch (FileNotFoundException fnfe) {
|
||||
logger.error("Error zipping file: " + fullFileName, fnfe.getMessage());
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
if ( ! e.getMessage().contains("duplicate") )
|
||||
logger.error("Error zipping file: " + fullFileName, e);
|
||||
return false;
|
||||
} finally {
|
||||
try {
|
||||
if ( bis != null )
|
||||
bis.close();
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue