changes for cris validation

src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java

@@ -5,6 +5,7 @@ import eu.dnetlib.api.functionality.ValidatorServiceException;
 import eu.dnetlib.domain.functionality.validator.StoredJob;
 import eu.dnetlib.repo.manager.domain.Constants;
 import eu.dnetlib.repo.manager.domain.JobsOfUser;
+import eu.dnetlib.repo.manager.utils.CrisValidatorUtils;
 import gr.uoa.di.driver.util.ServiceLocator;
 import org.apache.log4j.Logger;
 import org.eurocris.openaire.cris.validator.model.Job;
@@ -15,7 +16,6 @@ import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -56,6 +56,23 @@ public class MonitorServiceImpl implements MonitorService {
 
         LOGGER.debug("Getting jobs of user : " + user);
         LOGGER.debug(user + "/" + jobType + "/" + offset + "/" + dateFrom + "/" + dateTo + "/" + validationStatus + "/" + includeJobsTotal);
+
+        /////////////////////////////////////////////////////////////////////////////////////////
+        // FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
+        /////////////////////////////////////////////////////////////////////////////////////////
+        int crisOffset = 0;
+        int limitWithCris = 0;
+        if (offset.equals("0")) {
+            limitWithCris = crisJobs.getJobs(user, validationStatus).size();
+        } else {
+            crisOffset = crisJobs.getJobs(user, validationStatus).size();
+        }
+
+        offset = String.valueOf(Math.max(Integer.parseInt(offset) - crisOffset, 0));
+        limit = String.valueOf(Math.max(Integer.parseInt(limit) - limitWithCris, 0));
+        /////////////////////////////////////////////////////////////////////////////////////////
+        /////////////////////////////////////////////////////////////////////////////////////////
+
         JobsOfUser retJobs = new JobsOfUser();
         retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),
                 Integer.parseInt(limit), dateFrom, dateTo, validationStatus));
@@ -64,12 +81,12 @@ public class MonitorServiceImpl implements MonitorService {
             retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null));
             retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL));
             retJobs.setTotalJobsFailed(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_FAILED));
-            retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType,Constants.VALIDATION_JOB_STATUS_ONGOING));
+            retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_ONGOING));
         }
 
         //TODO fix status with new validator version
-        if(retJobs.getJobs() != null){
+        if (retJobs.getJobs() != null) {
-            for(StoredJob job :retJobs.getJobs()){
+            for (StoredJob job : retJobs.getJobs()) {
                 if (job.getContentJobStatus().equals("ongoing") || job.getUsageJobStatus().equals("ongoing")) {
                     job.setValidationStatus("ongoing");
                 } else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && job.getContentJobScore() > 50 && job.getUsageJobScore() > 50)
@@ -78,39 +95,42 @@ public class MonitorServiceImpl implements MonitorService {
                     job.setValidationStatus("successful");
                 } else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && (job.getContentJobScore() <= 50 || job.getUsageJobScore() <= 50))
                         || (job.getValidationType().equals("C") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("none") && job.getContentJobScore() <= 50)
-                        || (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50) ) {
+                        || (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50)) {
                     job.setValidationStatus("failed");
                 }
 
             }
         }
+
+
         /////////////////////////////////////////////////////////////////////////////////////////
         // FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
         /////////////////////////////////////////////////////////////////////////////////////////
-        List<StoredJob> jobs = new ArrayList<>();
+        if (crisOffset == 0) {
+            List<StoredJob> jobs = new ArrayList<>();
 
-        List<Job> cj = crisJobs.getJobs(user);
+            List<Job> cj = crisJobs.getJobs(user, validationStatus);
-        for (Job job : cj) {
+            for (Job job : cj) {
-
+                StoredJob sj = CrisValidatorUtils.convertJobToStoredJob(job);
-            StoredJob sj = converJobToStoredJob(job);
-
-
-            // filter out entries based on 'validationStatus'
-            if ("all".equals(validationStatus)) {
                 jobs.add(sj);
-            } else {
-                if (sj.getValidationStatus().equals(validationStatus)) {
-                    jobs.add(sj);
-                }
-            }
-        }
 
-        // add to CRIS Jan jobs all other jobs
+//                // filter out entries based on 'validationStatus'
-        if (retJobs.getJobs() != null) {
+//                if ("all".equals(validationStatus)) {
-            jobs.addAll(retJobs.getJobs());
+//                    jobs.add(sj);
+//                } else {
+//                    if (sj.getValidationStatus().equals(validationStatus)) {
+//                        jobs.add(sj);
+//                    }
+//                }
+            }
+
+            // add to CRIS Jan jobs all other jobs
+            if (retJobs.getJobs() != null) {
+                jobs.addAll(retJobs.getJobs());
+            }
+            // set all jobs back to retJobs
+            retJobs.setJobs(jobs);
         }
-        // set all jobs back to retJobs
-        retJobs.setJobs(jobs);
 
         // fix number of jobs
         if (Boolean.parseBoolean(includeJobsTotal)) {
@@ -127,7 +147,7 @@ public class MonitorServiceImpl implements MonitorService {
     }
 
     private int getJobsTotalNumberOfUser(String user, String jobType, String validationStatus) throws ValidatorServiceException {
-        return  getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
+        return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
     }
 
     @Override
@@ -135,6 +155,11 @@ public class MonitorServiceImpl implements MonitorService {
                                                 String jobType,
                                                 String validationStatus) throws JSONException {
         LOGGER.debug("Getting job with validation status : " + validationStatus);
+
+        if (jobType.equalsIgnoreCase("cris")) {
+            return crisJobs.getJobs(user, validationStatus).size();
+        }
+
         try {
             return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
         } catch (ValidatorServiceException e) {
@@ -160,45 +185,11 @@ public class MonitorServiceImpl implements MonitorService {
             // not a good way to do it but Job id field is string
             List<Job> cJobs = crisJobs.getAll().stream().filter(j -> j.getId().hashCode() == Integer.parseInt(jobId)).collect(Collectors.toList());
             if (!cJobs.isEmpty()) {
-                job = converJobToStoredJob(cJobs.get(0));
+                job = CrisValidatorUtils.convertJobToStoredJob(cJobs.get(0));
             }
         }
         /////////////////////////////////////////////////////////////////////////////////////////
         /////////////////////////////////////////////////////////////////////////////////////////
         return job;
     }
-
-    private StoredJob converJobToStoredJob(Job job) {
-        StoredJob sj = new StoredJob();
-        sj.setId(job.getId().hashCode());
-        sj.setValidationStatus(job.getStatus());
-        if (job.getDateFinished() != null) {
-            sj.setEnded(job.getDateFinished().toString());
-            sj.setDuration(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()).toString());
-        } else {
-            sj.setEnded("-");
-            sj.setDuration("-");
-        }
-        sj.setStarted(job.getDateStarted().toString());
-
-        sj.setUserEmail(job.getUser());
-        sj.setCris(true);
-        sj.setBaseUrl(job.getUrl());
-        sj.setJobType("CRIS Validation");
-
-        // TODO: status
-        sj.setValidationStatus(job.getStatus());
-        sj.setUsageJobStatus(job.getStatus());
-        sj.setContentJobStatus(job.getStatus());
-
-        // TODO: scores
-//            sj.setFilteredScores();
-        sj.setContentJobScore(job.getScore());
-        sj.setUsageJobScore(job.getScore());
-
-        sj.setValidationType("CRIS Validation");
-
-        return sj;
-    }
-
 }

src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java

@@ -2,13 +2,11 @@ package eu.dnetlib.repo.manager.service;
 
 import eu.dnetlib.api.functionality.ValidatorServiceException;
 import eu.dnetlib.domain.data.RepositoryInterface;
-import eu.dnetlib.domain.functionality.validator.JobForValidation;
+import eu.dnetlib.domain.functionality.validator.*;
-import eu.dnetlib.domain.functionality.validator.Rule;
-import eu.dnetlib.domain.functionality.validator.RuleSet;
-import eu.dnetlib.domain.functionality.validator.StoredJob;
 import eu.dnetlib.repo.manager.domain.Constants;
 import eu.dnetlib.repo.manager.domain.InterfaceInformation;
 import eu.dnetlib.repo.manager.domain.ValidationServiceException;
+import eu.dnetlib.repo.manager.utils.CrisValidatorUtils;
 import eu.dnetlib.repo.manager.utils.OaiTools;
 import gr.uoa.di.driver.util.ServiceLocator;
 import org.apache.log4j.Logger;
@@ -167,8 +165,9 @@ public class ValidatorServiceImpl implements ValidatorService {
         for (List<RuleSet> ruleSets : this.rulesetMap.values()) {
             for (RuleSet rSet : ruleSets) {
                 if (rSet.getName().equalsIgnoreCase("CRIS Validation")) {
-
+                    crisValidatorExecutor.submit(job.getBaseUrl(), job.getUserEmail());
-                }else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){
+                    return new ResponseEntity<>("OK",HttpStatus.OK);
+                } else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){
                     ruleSet = rSet;
                     break;
                 }
@@ -259,33 +258,7 @@ public class ValidatorServiceImpl implements ValidatorService {
         }
         List<Job> cj = crisJobs.getJobs(user);
         for (Job job : cj) {
-            StoredJob sj = new StoredJob();
+            jobs.add(CrisValidatorUtils.convertJobToStoredJob(job));
-
-            if (job.getDateFinished() != null) {
-                sj.setEnded(job.getDateFinished().toString());
-                sj.setDuration(Long.toString(job.getDateFinished().getTime() - job.getDateStarted().getTime()) + " seconds");
-            } else {
-                sj.setEnded("-");
-                sj.setDuration("-");
-            }
-            sj.setStarted(job.getDateStarted().toString());
-
-            sj.setUserEmail(job.getUser());
-            sj.setCris(true);
-            sj.setBaseUrl(job.getUrl());
-            sj.setJobType("CRIS Validation");
-
-            // TODO: status
-            sj.setValidationStatus(job.getStatus());
-            sj.setUsageJobStatus(job.getStatus());
-            sj.setContentJobStatus(job.getStatus());
-
-            // TODO: scores
-//            sj.setFilteredScores();
-            sj.setContentJobScore(job.getScore());
-            sj.setUsageJobScore(job.getScore());
-
-            jobs.add(sj);
         }
         /////////////////////////////////////////////////////////////////////////////////////////
         /////////////////////////////////////////////////////////////////////////////////////////
@@ -319,5 +292,4 @@ public class ValidatorServiceImpl implements ValidatorService {
         return getValidationService().getJobSummary(repositoryService.getRepositoryInterface(repoId).stream().map(RepositoryInterface::getBaseUrl).collect(Collectors.toList()),limit);
     }
 
-
 }

src/main/java/eu/dnetlib/repo/manager/utils/CrisValidatorUtils.java

@@ -0,0 +1,82 @@
+package eu.dnetlib.repo.manager.utils;
+
+import eu.dnetlib.domain.functionality.validator.JobResultEntry;
+import eu.dnetlib.domain.functionality.validator.StoredJob;
+import org.eurocris.openaire.cris.validator.model.Job;
+import org.eurocris.openaire.cris.validator.model.RuleResults;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+public class CrisValidatorUtils {
+
+    private static final DateFormat DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    private static final DateFormat DURATION_FORMATTER = new SimpleDateFormat("HH 'hours' mm 'min' ss 'sec'");
+
+    public static StoredJob convertJobToStoredJob(Job job) {
+        StoredJob sj = new StoredJob();
+        sj.setId(job.getId().hashCode());
+        sj.setValidationStatus(job.getStatus());
+        if (job.getDateFinished() != null) {
+            sj.setEnded(DATE_FORMATTER.format(job.getDateFinished()));
+            sj.setDuration(DURATION_FORMATTER.format(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime())));
+        } else {
+            sj.setEnded("-");
+            sj.setDuration("-");
+        }
+        sj.setStarted(DATE_FORMATTER.format(job.getDateStarted()));
+
+        sj.setUserEmail(job.getUser());
+        sj.setCris(true);
+        sj.setBaseUrl(job.getUrl());
+        sj.setValidationType("CU");
+        sj.setJobType("CRIS Validation");
+        sj.setGuidelinesShortName("For CRIS Managers 1.1");
+        sj.setValidationSet("-");
+
+        // Set status
+        sj.setValidationStatus(job.getStatus());
+        sj.setUsageJobStatus(job.getUsageJobStatus());
+        sj.setContentJobStatus(job.getContentJobStatus());
+
+        // Set scores
+//            sj.setFilteredScores(); // TODO what is this?
+        sj.setContentJobScore(job.getContentScore());
+        sj.setUsageJobScore(job.getUsageScore());
+
+        sj.setResultEntries(CrisValidatorUtils.crisResultsToJobEntries(job));
+        sj.setRecordsTested(job.getRecordsTested());
+
+        return sj;
+    }
+
+    public static List<JobResultEntry> crisResultsToJobEntries(Job crisJob) {
+        List<JobResultEntry> jobResultEntries = new ArrayList<>();
+        for (RuleResults ruleResults : crisJob.getRuleResults()) {
+            JobResultEntry jobResultEntry = new JobResultEntry();
+            jobResultEntry.setName(ruleResults.getRuleMethodName());
+            jobResultEntry.setRuleId(ruleResults.getRuleId());
+            jobResultEntry.setMandatory(true);
+            jobResultEntry.setWeight(Math.round(ruleResults.getWeight()));
+            jobResultEntry.setType(ruleResults.getType());
+            jobResultEntry.setHasErrors(false);
+            if (ruleResults.getCount() == 0) {
+                jobResultEntry.setSuccesses("-");
+            } else {
+                jobResultEntry.setSuccesses(ruleResults.getCount() - ruleResults.getFailed() + "/" + ruleResults.getCount());
+            }
+            if (ruleResults.getErrorMessages() != null && !ruleResults.getErrorMessages().isEmpty()) {
+                jobResultEntry.setErrors(ruleResults.getErrorMessages());
+                jobResultEntry.setHasErrors(true);
+            }
+            jobResultEntries.add(jobResultEntry);
+        }
+        return jobResultEntries;
+    }
+
+    private CrisValidatorUtils() {
+    }
+}