From c0cbbbe9ef61ac426f44dbe64b541acefff6b69d Mon Sep 17 00:00:00 2001 From: spyroukon Date: Wed, 18 Mar 2020 22:39:25 +0000 Subject: [PATCH] changes for cris validation --- .../manager/service/MonitorServiceImpl.java | 113 ++++++++---------- .../manager/service/ValidatorServiceImpl.java | 40 +------ .../manager/utils/CrisValidatorUtils.java | 82 +++++++++++++ 3 files changed, 140 insertions(+), 95 deletions(-) create mode 100644 src/main/java/eu/dnetlib/repo/manager/utils/CrisValidatorUtils.java diff --git a/src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java b/src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java index 3b07ea5..0670eea 100644 --- a/src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java +++ b/src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java @@ -5,6 +5,7 @@ import eu.dnetlib.api.functionality.ValidatorServiceException; import eu.dnetlib.domain.functionality.validator.StoredJob; import eu.dnetlib.repo.manager.domain.Constants; import eu.dnetlib.repo.manager.domain.JobsOfUser; +import eu.dnetlib.repo.manager.utils.CrisValidatorUtils; import gr.uoa.di.driver.util.ServiceLocator; import org.apache.log4j.Logger; import org.eurocris.openaire.cris.validator.model.Job; @@ -15,7 +16,6 @@ import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.util.ArrayList; -import java.util.Date; import java.util.List; import java.util.stream.Collectors; @@ -56,6 +56,23 @@ public class MonitorServiceImpl implements MonitorService { LOGGER.debug("Getting jobs of user : " + user); LOGGER.debug(user + "/" + jobType + "/" + offset + "/" + dateFrom + "/" + dateTo + "/" + validationStatus + "/" + includeJobsTotal); + + ///////////////////////////////////////////////////////////////////////////////////////// + // FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly // + ///////////////////////////////////////////////////////////////////////////////////////// + int crisOffset = 0; + int limitWithCris = 0; + if (offset.equals("0")) { + limitWithCris = crisJobs.getJobs(user, validationStatus).size(); + } else { + crisOffset = crisJobs.getJobs(user, validationStatus).size(); + } + + offset = String.valueOf(Math.max(Integer.parseInt(offset) - crisOffset, 0)); + limit = String.valueOf(Math.max(Integer.parseInt(limit) - limitWithCris, 0)); + ///////////////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////////////// + JobsOfUser retJobs = new JobsOfUser(); retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus)); @@ -64,12 +81,12 @@ public class MonitorServiceImpl implements MonitorService { retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null)); retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL)); retJobs.setTotalJobsFailed(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_FAILED)); - retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType,Constants.VALIDATION_JOB_STATUS_ONGOING)); + retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_ONGOING)); } //TODO fix status with new validator version - if(retJobs.getJobs() != null){ - for(StoredJob job :retJobs.getJobs()){ + if (retJobs.getJobs() != null) { + for (StoredJob job : retJobs.getJobs()) { if (job.getContentJobStatus().equals("ongoing") || job.getUsageJobStatus().equals("ongoing")) { job.setValidationStatus("ongoing"); } else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && job.getContentJobScore() > 50 && job.getUsageJobScore() > 50) @@ -78,39 +95,42 @@ public class MonitorServiceImpl implements MonitorService { job.setValidationStatus("successful"); } else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && (job.getContentJobScore() <= 50 || job.getUsageJobScore() <= 50)) || (job.getValidationType().equals("C") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("none") && job.getContentJobScore() <= 50) - || (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50) ) { + || (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50)) { job.setValidationStatus("failed"); } } } + + ///////////////////////////////////////////////////////////////////////////////////////// // FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly // ///////////////////////////////////////////////////////////////////////////////////////// - List jobs = new ArrayList<>(); + if (crisOffset == 0) { + List jobs = new ArrayList<>(); - List cj = crisJobs.getJobs(user); - for (Job job : cj) { - - StoredJob sj = converJobToStoredJob(job); - - - // filter out entries based on 'validationStatus' - if ("all".equals(validationStatus)) { + List cj = crisJobs.getJobs(user, validationStatus); + for (Job job : cj) { + StoredJob sj = CrisValidatorUtils.convertJobToStoredJob(job); jobs.add(sj); - } else { - if (sj.getValidationStatus().equals(validationStatus)) { - jobs.add(sj); - } - } - } - // add to CRIS Jan jobs all other jobs - if (retJobs.getJobs() != null) { - jobs.addAll(retJobs.getJobs()); +// // filter out entries based on 'validationStatus' +// if ("all".equals(validationStatus)) { +// jobs.add(sj); +// } else { +// if (sj.getValidationStatus().equals(validationStatus)) { +// jobs.add(sj); +// } +// } + } + + // add to CRIS Jan jobs all other jobs + if (retJobs.getJobs() != null) { + jobs.addAll(retJobs.getJobs()); + } + // set all jobs back to retJobs + retJobs.setJobs(jobs); } - // set all jobs back to retJobs - retJobs.setJobs(jobs); // fix number of jobs if (Boolean.parseBoolean(includeJobsTotal)) { @@ -127,7 +147,7 @@ public class MonitorServiceImpl implements MonitorService { } private int getJobsTotalNumberOfUser(String user, String jobType, String validationStatus) throws ValidatorServiceException { - return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus); + return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus); } @Override @@ -135,6 +155,11 @@ public class MonitorServiceImpl implements MonitorService { String jobType, String validationStatus) throws JSONException { LOGGER.debug("Getting job with validation status : " + validationStatus); + + if (jobType.equalsIgnoreCase("cris")) { + return crisJobs.getJobs(user, validationStatus).size(); + } + try { return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus); } catch (ValidatorServiceException e) { @@ -160,45 +185,11 @@ public class MonitorServiceImpl implements MonitorService { // not a good way to do it but Job id field is string List cJobs = crisJobs.getAll().stream().filter(j -> j.getId().hashCode() == Integer.parseInt(jobId)).collect(Collectors.toList()); if (!cJobs.isEmpty()) { - job = converJobToStoredJob(cJobs.get(0)); + job = CrisValidatorUtils.convertJobToStoredJob(cJobs.get(0)); } } ///////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////// return job; } - - private StoredJob converJobToStoredJob(Job job) { - StoredJob sj = new StoredJob(); - sj.setId(job.getId().hashCode()); - sj.setValidationStatus(job.getStatus()); - if (job.getDateFinished() != null) { - sj.setEnded(job.getDateFinished().toString()); - sj.setDuration(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()).toString()); - } else { - sj.setEnded("-"); - sj.setDuration("-"); - } - sj.setStarted(job.getDateStarted().toString()); - - sj.setUserEmail(job.getUser()); - sj.setCris(true); - sj.setBaseUrl(job.getUrl()); - sj.setJobType("CRIS Validation"); - - // TODO: status - sj.setValidationStatus(job.getStatus()); - sj.setUsageJobStatus(job.getStatus()); - sj.setContentJobStatus(job.getStatus()); - - // TODO: scores -// sj.setFilteredScores(); - sj.setContentJobScore(job.getScore()); - sj.setUsageJobScore(job.getScore()); - - sj.setValidationType("CRIS Validation"); - - return sj; - } - } diff --git a/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java b/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java index 2e7f41f..8c62afe 100644 --- a/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java +++ b/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java @@ -2,13 +2,11 @@ package eu.dnetlib.repo.manager.service; import eu.dnetlib.api.functionality.ValidatorServiceException; import eu.dnetlib.domain.data.RepositoryInterface; -import eu.dnetlib.domain.functionality.validator.JobForValidation; -import eu.dnetlib.domain.functionality.validator.Rule; -import eu.dnetlib.domain.functionality.validator.RuleSet; -import eu.dnetlib.domain.functionality.validator.StoredJob; +import eu.dnetlib.domain.functionality.validator.*; import eu.dnetlib.repo.manager.domain.Constants; import eu.dnetlib.repo.manager.domain.InterfaceInformation; import eu.dnetlib.repo.manager.domain.ValidationServiceException; +import eu.dnetlib.repo.manager.utils.CrisValidatorUtils; import eu.dnetlib.repo.manager.utils.OaiTools; import gr.uoa.di.driver.util.ServiceLocator; import org.apache.log4j.Logger; @@ -167,8 +165,9 @@ public class ValidatorServiceImpl implements ValidatorService { for (List ruleSets : this.rulesetMap.values()) { for (RuleSet rSet : ruleSets) { if (rSet.getName().equalsIgnoreCase("CRIS Validation")) { - - }else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){ + crisValidatorExecutor.submit(job.getBaseUrl(), job.getUserEmail()); + return new ResponseEntity<>("OK",HttpStatus.OK); + } else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){ ruleSet = rSet; break; } @@ -259,33 +258,7 @@ public class ValidatorServiceImpl implements ValidatorService { } List cj = crisJobs.getJobs(user); for (Job job : cj) { - StoredJob sj = new StoredJob(); - - if (job.getDateFinished() != null) { - sj.setEnded(job.getDateFinished().toString()); - sj.setDuration(Long.toString(job.getDateFinished().getTime() - job.getDateStarted().getTime()) + " seconds"); - } else { - sj.setEnded("-"); - sj.setDuration("-"); - } - sj.setStarted(job.getDateStarted().toString()); - - sj.setUserEmail(job.getUser()); - sj.setCris(true); - sj.setBaseUrl(job.getUrl()); - sj.setJobType("CRIS Validation"); - - // TODO: status - sj.setValidationStatus(job.getStatus()); - sj.setUsageJobStatus(job.getStatus()); - sj.setContentJobStatus(job.getStatus()); - - // TODO: scores -// sj.setFilteredScores(); - sj.setContentJobScore(job.getScore()); - sj.setUsageJobScore(job.getScore()); - - jobs.add(sj); + jobs.add(CrisValidatorUtils.convertJobToStoredJob(job)); } ///////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////// @@ -319,5 +292,4 @@ public class ValidatorServiceImpl implements ValidatorService { return getValidationService().getJobSummary(repositoryService.getRepositoryInterface(repoId).stream().map(RepositoryInterface::getBaseUrl).collect(Collectors.toList()),limit); } - } diff --git a/src/main/java/eu/dnetlib/repo/manager/utils/CrisValidatorUtils.java b/src/main/java/eu/dnetlib/repo/manager/utils/CrisValidatorUtils.java new file mode 100644 index 0000000..7eee9e3 --- /dev/null +++ b/src/main/java/eu/dnetlib/repo/manager/utils/CrisValidatorUtils.java @@ -0,0 +1,82 @@ +package eu.dnetlib.repo.manager.utils; + +import eu.dnetlib.domain.functionality.validator.JobResultEntry; +import eu.dnetlib.domain.functionality.validator.StoredJob; +import org.eurocris.openaire.cris.validator.model.Job; +import org.eurocris.openaire.cris.validator.model.RuleResults; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class CrisValidatorUtils { + + private static final DateFormat DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static final DateFormat DURATION_FORMATTER = new SimpleDateFormat("HH 'hours' mm 'min' ss 'sec'"); + + public static StoredJob convertJobToStoredJob(Job job) { + StoredJob sj = new StoredJob(); + sj.setId(job.getId().hashCode()); + sj.setValidationStatus(job.getStatus()); + if (job.getDateFinished() != null) { + sj.setEnded(DATE_FORMATTER.format(job.getDateFinished())); + sj.setDuration(DURATION_FORMATTER.format(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()))); + } else { + sj.setEnded("-"); + sj.setDuration("-"); + } + sj.setStarted(DATE_FORMATTER.format(job.getDateStarted())); + + sj.setUserEmail(job.getUser()); + sj.setCris(true); + sj.setBaseUrl(job.getUrl()); + sj.setValidationType("CU"); + sj.setJobType("CRIS Validation"); + sj.setGuidelinesShortName("For CRIS Managers 1.1"); + sj.setValidationSet("-"); + + // Set status + sj.setValidationStatus(job.getStatus()); + sj.setUsageJobStatus(job.getUsageJobStatus()); + sj.setContentJobStatus(job.getContentJobStatus()); + + // Set scores +// sj.setFilteredScores(); // TODO what is this? + sj.setContentJobScore(job.getContentScore()); + sj.setUsageJobScore(job.getUsageScore()); + + sj.setResultEntries(CrisValidatorUtils.crisResultsToJobEntries(job)); + sj.setRecordsTested(job.getRecordsTested()); + + return sj; + } + + public static List crisResultsToJobEntries(Job crisJob) { + List jobResultEntries = new ArrayList<>(); + for (RuleResults ruleResults : crisJob.getRuleResults()) { + JobResultEntry jobResultEntry = new JobResultEntry(); + jobResultEntry.setName(ruleResults.getRuleMethodName()); + jobResultEntry.setRuleId(ruleResults.getRuleId()); + jobResultEntry.setMandatory(true); + jobResultEntry.setWeight(Math.round(ruleResults.getWeight())); + jobResultEntry.setType(ruleResults.getType()); + jobResultEntry.setHasErrors(false); + if (ruleResults.getCount() == 0) { + jobResultEntry.setSuccesses("-"); + } else { + jobResultEntry.setSuccesses(ruleResults.getCount() - ruleResults.getFailed() + "/" + ruleResults.getCount()); + } + if (ruleResults.getErrorMessages() != null && !ruleResults.getErrorMessages().isEmpty()) { + jobResultEntry.setErrors(ruleResults.getErrorMessages()); + jobResultEntry.setHasErrors(true); + } + jobResultEntries.add(jobResultEntry); + } + return jobResultEntries; + } + + private CrisValidatorUtils() { + } +}