MaDgIK
/
uoa-repository-manager-service
13
2
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

changes for cris validation

This commit is contained in:
Konstantinos Spyrou 2020-03-18 22:39:25 +00:00
parent 777d87c564
commit c0cbbbe9ef
3 changed files with 140 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java
View File

@ -5,6 +5,7 @@ import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import eu.dnetlib.repo.manager.domain.Constants;
import eu.dnetlib.repo.manager.domain.JobsOfUser;
import eu.dnetlib.repo.manager.utils.CrisValidatorUtils;
import gr.uoa.di.driver.util.ServiceLocator;
import org.apache.log4j.Logger;
import org.eurocris.openaire.cris.validator.model.Job;
@ -15,7 +16,6 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
@ -56,6 +56,23 @@ public class MonitorServiceImpl implements MonitorService {
LOGGER.debug("Getting jobs of user : " + user);
LOGGER.debug(user + "/" + jobType + "/" + offset + "/" + dateFrom + "/" + dateTo + "/" + validationStatus + "/" + includeJobsTotal);
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
int crisOffset = 0;
int limitWithCris = 0;
if (offset.equals("0")) {
limitWithCris = crisJobs.getJobs(user, validationStatus).size();
} else {
crisOffset = crisJobs.getJobs(user, validationStatus).size();
}
offset = String.valueOf(Math.max(Integer.parseInt(offset) - crisOffset, 0));
limit = String.valueOf(Math.max(Integer.parseInt(limit) - limitWithCris, 0));
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
JobsOfUser retJobs = new JobsOfUser();
retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),
Integer.parseInt(limit), dateFrom, dateTo, validationStatus));
@ -64,12 +81,12 @@ public class MonitorServiceImpl implements MonitorService {
retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null));
retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL));
retJobs.setTotalJobsFailed(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_FAILED));
retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType,Constants.VALIDATION_JOB_STATUS_ONGOING));
retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_ONGOING));
}
//TODO fix status with new validator version
if(retJobs.getJobs() != null){
for(StoredJob job :retJobs.getJobs()){
if (retJobs.getJobs() != null) {
for (StoredJob job : retJobs.getJobs()) {
if (job.getContentJobStatus().equals("ongoing") || job.getUsageJobStatus().equals("ongoing")) {
job.setValidationStatus("ongoing");
} else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && job.getContentJobScore() > 50 && job.getUsageJobScore() > 50)
@ -78,39 +95,42 @@ public class MonitorServiceImpl implements MonitorService {
job.setValidationStatus("successful");
} else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && (job.getContentJobScore() <= 50 || job.getUsageJobScore() <= 50))
|| (job.getValidationType().equals("C") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("none") && job.getContentJobScore() <= 50)
|| (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50) ) {
|| (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50)) {
job.setValidationStatus("failed");
}
}
}
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
List<StoredJob> jobs = new ArrayList<>();
if (crisOffset == 0) {
List<StoredJob> jobs = new ArrayList<>();
List<Job> cj = crisJobs.getJobs(user);
for (Job job : cj) {
StoredJob sj = converJobToStoredJob(job);
// filter out entries based on 'validationStatus'
if ("all".equals(validationStatus)) {
List<Job> cj = crisJobs.getJobs(user, validationStatus);
for (Job job : cj) {
StoredJob sj = CrisValidatorUtils.convertJobToStoredJob(job);
jobs.add(sj);
} else {
if (sj.getValidationStatus().equals(validationStatus)) {
jobs.add(sj);
}
}
}
// add to CRIS Jan jobs all other jobs
if (retJobs.getJobs() != null) {
jobs.addAll(retJobs.getJobs());
// // filter out entries based on 'validationStatus'
// if ("all".equals(validationStatus)) {
// jobs.add(sj);
// } else {
// if (sj.getValidationStatus().equals(validationStatus)) {
// jobs.add(sj);
// }
// }
}
// add to CRIS Jan jobs all other jobs
if (retJobs.getJobs() != null) {
jobs.addAll(retJobs.getJobs());
}
// set all jobs back to retJobs
retJobs.setJobs(jobs);
}
// set all jobs back to retJobs
retJobs.setJobs(jobs);
// fix number of jobs
if (Boolean.parseBoolean(includeJobsTotal)) {
@ -127,7 +147,7 @@ public class MonitorServiceImpl implements MonitorService {
}
private int getJobsTotalNumberOfUser(String user, String jobType, String validationStatus) throws ValidatorServiceException {
return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
}
@Override
@ -135,6 +155,11 @@ public class MonitorServiceImpl implements MonitorService {
String jobType,
String validationStatus) throws JSONException {
LOGGER.debug("Getting job with validation status : " + validationStatus);
if (jobType.equalsIgnoreCase("cris")) {
return crisJobs.getJobs(user, validationStatus).size();
}
try {
return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
} catch (ValidatorServiceException e) {
@ -160,45 +185,11 @@ public class MonitorServiceImpl implements MonitorService {
// not a good way to do it but Job id field is string
List<Job> cJobs = crisJobs.getAll().stream().filter(j -> j.getId().hashCode() == Integer.parseInt(jobId)).collect(Collectors.toList());
if (!cJobs.isEmpty()) {
job = converJobToStoredJob(cJobs.get(0));
job = CrisValidatorUtils.convertJobToStoredJob(cJobs.get(0));
}
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
return job;
}
private StoredJob converJobToStoredJob(Job job) {
StoredJob sj = new StoredJob();
sj.setId(job.getId().hashCode());
sj.setValidationStatus(job.getStatus());
if (job.getDateFinished() != null) {
sj.setEnded(job.getDateFinished().toString());
sj.setDuration(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()).toString());
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(job.getDateStarted().toString());
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setJobType("CRIS Validation");
// TODO: status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getStatus());
sj.setContentJobStatus(job.getStatus());
// TODO: scores
// sj.setFilteredScores();
sj.setContentJobScore(job.getScore());
sj.setUsageJobScore(job.getScore());
sj.setValidationType("CRIS Validation");
return sj;
}
}

40
src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java
View File

@ -2,13 +2,11 @@ package eu.dnetlib.repo.manager.service;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.data.RepositoryInterface;
import eu.dnetlib.domain.functionality.validator.JobForValidation;
import eu.dnetlib.domain.functionality.validator.Rule;
import eu.dnetlib.domain.functionality.validator.RuleSet;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import eu.dnetlib.domain.functionality.validator.*;
import eu.dnetlib.repo.manager.domain.Constants;
import eu.dnetlib.repo.manager.domain.InterfaceInformation;
import eu.dnetlib.repo.manager.domain.ValidationServiceException;
import eu.dnetlib.repo.manager.utils.CrisValidatorUtils;
import eu.dnetlib.repo.manager.utils.OaiTools;
import gr.uoa.di.driver.util.ServiceLocator;
import org.apache.log4j.Logger;
@ -167,8 +165,9 @@ public class ValidatorServiceImpl implements ValidatorService {
for (List<RuleSet> ruleSets : this.rulesetMap.values()) {
for (RuleSet rSet : ruleSets) {
if (rSet.getName().equalsIgnoreCase("CRIS Validation")) {
}else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){
crisValidatorExecutor.submit(job.getBaseUrl(), job.getUserEmail());
return new ResponseEntity<>("OK",HttpStatus.OK);
} else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){
ruleSet = rSet;
break;
}
@ -259,33 +258,7 @@ public class ValidatorServiceImpl implements ValidatorService {
}
List<Job> cj = crisJobs.getJobs(user);
for (Job job : cj) {
StoredJob sj = new StoredJob();
if (job.getDateFinished() != null) {
sj.setEnded(job.getDateFinished().toString());
sj.setDuration(Long.toString(job.getDateFinished().getTime() - job.getDateStarted().getTime()) + " seconds");
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(job.getDateStarted().toString());
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setJobType("CRIS Validation");
// TODO: status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getStatus());
sj.setContentJobStatus(job.getStatus());
// TODO: scores
// sj.setFilteredScores();
sj.setContentJobScore(job.getScore());
sj.setUsageJobScore(job.getScore());
jobs.add(sj);
jobs.add(CrisValidatorUtils.convertJobToStoredJob(job));
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
@ -319,5 +292,4 @@ public class ValidatorServiceImpl implements ValidatorService {
return getValidationService().getJobSummary(repositoryService.getRepositoryInterface(repoId).stream().map(RepositoryInterface::getBaseUrl).collect(Collectors.toList()),limit);
}
}

82
src/main/java/eu/dnetlib/repo/manager/utils/CrisValidatorUtils.java Normal file
View File

@ -0,0 +1,82 @@
package eu.dnetlib.repo.manager.utils;
import eu.dnetlib.domain.functionality.validator.JobResultEntry;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import org.eurocris.openaire.cris.validator.model.Job;
import org.eurocris.openaire.cris.validator.model.RuleResults;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
public class CrisValidatorUtils {
private static final DateFormat DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private static final DateFormat DURATION_FORMATTER = new SimpleDateFormat("HH 'hours' mm 'min' ss 'sec'");
public static StoredJob convertJobToStoredJob(Job job) {
StoredJob sj = new StoredJob();
sj.setId(job.getId().hashCode());
sj.setValidationStatus(job.getStatus());
if (job.getDateFinished() != null) {
sj.setEnded(DATE_FORMATTER.format(job.getDateFinished()));
sj.setDuration(DURATION_FORMATTER.format(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime())));
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(DATE_FORMATTER.format(job.getDateStarted()));
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setValidationType("CU");
sj.setJobType("CRIS Validation");
sj.setGuidelinesShortName("For CRIS Managers 1.1");
sj.setValidationSet("-");
// Set status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getUsageJobStatus());
sj.setContentJobStatus(job.getContentJobStatus());
// Set scores
// sj.setFilteredScores(); // TODO what is this?
sj.setContentJobScore(job.getContentScore());
sj.setUsageJobScore(job.getUsageScore());
sj.setResultEntries(CrisValidatorUtils.crisResultsToJobEntries(job));
sj.setRecordsTested(job.getRecordsTested());
return sj;
}
public static List<JobResultEntry> crisResultsToJobEntries(Job crisJob) {
List<JobResultEntry> jobResultEntries = new ArrayList<>();
for (RuleResults ruleResults : crisJob.getRuleResults()) {
JobResultEntry jobResultEntry = new JobResultEntry();
jobResultEntry.setName(ruleResults.getRuleMethodName());
jobResultEntry.setRuleId(ruleResults.getRuleId());
jobResultEntry.setMandatory(true);
jobResultEntry.setWeight(Math.round(ruleResults.getWeight()));
jobResultEntry.setType(ruleResults.getType());
jobResultEntry.setHasErrors(false);
if (ruleResults.getCount() == 0) {
jobResultEntry.setSuccesses("-");
} else {
jobResultEntry.setSuccesses(ruleResults.getCount() - ruleResults.getFailed() + "/" + ruleResults.getCount());
}
if (ruleResults.getErrorMessages() != null && !ruleResults.getErrorMessages().isEmpty()) {
jobResultEntry.setErrors(ruleResults.getErrorMessages());
jobResultEntry.setHasErrors(true);
}
jobResultEntries.add(jobResultEntry);
}
return jobResultEntries;
}
private CrisValidatorUtils() {
}
}