MaDgIK
/
uoa-repository-manager-service
13
2
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

changes for cris validation

This commit is contained in:
Konstantinos Spyrou 2020-03-18 22:39:25 +00:00
parent 777d87c564
commit c0cbbbe9ef
3 changed files with 140 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java
View File

@ -5,6 +5,7 @@ import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import eu.dnetlib.repo.manager.domain.Constants;
import eu.dnetlib.repo.manager.domain.JobsOfUser;
import eu.dnetlib.repo.manager.utils.CrisValidatorUtils;
import gr.uoa.di.driver.util.ServiceLocator;
import org.apache.log4j.Logger;
import org.eurocris.openaire.cris.validator.model.Job;
@ -15,7 +16,6 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
@ -56,6 +56,23 @@ public class MonitorServiceImpl implements MonitorService {
LOGGER.debug("Getting jobs of user : " + user);
LOGGER.debug(user + "/" + jobType + "/" + offset + "/" + dateFrom + "/" + dateTo + "/" + validationStatus + "/" + includeJobsTotal);
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
int crisOffset = 0;
int limitWithCris = 0;
if (offset.equals("0")) {
limitWithCris = crisJobs.getJobs(user, validationStatus).size();
} else {
crisOffset = crisJobs.getJobs(user, validationStatus).size();
}
offset = String.valueOf(Math.max(Integer.parseInt(offset) - crisOffset, 0));
limit = String.valueOf(Math.max(Integer.parseInt(limit) - limitWithCris, 0));
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
JobsOfUser retJobs = new JobsOfUser();
retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),
Integer.parseInt(limit), dateFrom, dateTo, validationStatus));
@ -84,25 +101,27 @@ public class MonitorServiceImpl implements MonitorService {
}
}
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
if (crisOffset == 0) {
List<StoredJob> jobs = new ArrayList<>();
List<Job> cj = crisJobs.getJobs(user);
List<Job> cj = crisJobs.getJobs(user, validationStatus);
for (Job job : cj) {
StoredJob sj = converJobToStoredJob(job);
// filter out entries based on 'validationStatus'
if ("all".equals(validationStatus)) {
StoredJob sj = CrisValidatorUtils.convertJobToStoredJob(job);
jobs.add(sj);
} else {
if (sj.getValidationStatus().equals(validationStatus)) {
jobs.add(sj);
}
}
// // filter out entries based on 'validationStatus'
// if ("all".equals(validationStatus)) {
// jobs.add(sj);
// } else {
// if (sj.getValidationStatus().equals(validationStatus)) {
// jobs.add(sj);
// }
// }
}
// add to CRIS Jan jobs all other jobs
@ -111,6 +130,7 @@ public class MonitorServiceImpl implements MonitorService {
}
// set all jobs back to retJobs
retJobs.setJobs(jobs);
}
// fix number of jobs
if (Boolean.parseBoolean(includeJobsTotal)) {
@ -135,6 +155,11 @@ public class MonitorServiceImpl implements MonitorService {
String jobType,
String validationStatus) throws JSONException {
LOGGER.debug("Getting job with validation status : " + validationStatus);
if (jobType.equalsIgnoreCase("cris")) {
return crisJobs.getJobs(user, validationStatus).size();
}
try {
return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
} catch (ValidatorServiceException e) {
@ -160,45 +185,11 @@ public class MonitorServiceImpl implements MonitorService {
// not a good way to do it but Job id field is string
List<Job> cJobs = crisJobs.getAll().stream().filter(j -> j.getId().hashCode() == Integer.parseInt(jobId)).collect(Collectors.toList());
if (!cJobs.isEmpty()) {
job = converJobToStoredJob(cJobs.get(0));
job = CrisValidatorUtils.convertJobToStoredJob(cJobs.get(0));
}
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
return job;
}
private StoredJob converJobToStoredJob(Job job) {
StoredJob sj = new StoredJob();
sj.setId(job.getId().hashCode());
sj.setValidationStatus(job.getStatus());
if (job.getDateFinished() != null) {
sj.setEnded(job.getDateFinished().toString());
sj.setDuration(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()).toString());
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(job.getDateStarted().toString());
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setJobType("CRIS Validation");
// TODO: status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getStatus());
sj.setContentJobStatus(job.getStatus());
// TODO: scores
// sj.setFilteredScores();
sj.setContentJobScore(job.getScore());
sj.setUsageJobScore(job.getScore());
sj.setValidationType("CRIS Validation");
return sj;
}
}

38
src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java
View File

@ -2,13 +2,11 @@ package eu.dnetlib.repo.manager.service;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.data.RepositoryInterface;
import eu.dnetlib.domain.functionality.validator.JobForValidation;
import eu.dnetlib.domain.functionality.validator.Rule;
import eu.dnetlib.domain.functionality.validator.RuleSet;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import eu.dnetlib.domain.functionality.validator.*;
import eu.dnetlib.repo.manager.domain.Constants;
import eu.dnetlib.repo.manager.domain.InterfaceInformation;
import eu.dnetlib.repo.manager.domain.ValidationServiceException;
import eu.dnetlib.repo.manager.utils.CrisValidatorUtils;
import eu.dnetlib.repo.manager.utils.OaiTools;
import gr.uoa.di.driver.util.ServiceLocator;
import org.apache.log4j.Logger;
@ -167,7 +165,8 @@ public class ValidatorServiceImpl implements ValidatorService {
for (List<RuleSet> ruleSets : this.rulesetMap.values()) {
for (RuleSet rSet : ruleSets) {
if (rSet.getName().equalsIgnoreCase("CRIS Validation")) {
crisValidatorExecutor.submit(job.getBaseUrl(), job.getUserEmail());
return new ResponseEntity<>("OK",HttpStatus.OK);
} else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){
ruleSet = rSet;
break;
@ -259,33 +258,7 @@ public class ValidatorServiceImpl implements ValidatorService {
}
List<Job> cj = crisJobs.getJobs(user);
for (Job job : cj) {
StoredJob sj = new StoredJob();
if (job.getDateFinished() != null) {
sj.setEnded(job.getDateFinished().toString());
sj.setDuration(Long.toString(job.getDateFinished().getTime() - job.getDateStarted().getTime()) + " seconds");
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(job.getDateStarted().toString());
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setJobType("CRIS Validation");
// TODO: status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getStatus());
sj.setContentJobStatus(job.getStatus());
// TODO: scores
// sj.setFilteredScores();
sj.setContentJobScore(job.getScore());
sj.setUsageJobScore(job.getScore());
jobs.add(sj);
jobs.add(CrisValidatorUtils.convertJobToStoredJob(job));
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
@ -319,5 +292,4 @@ public class ValidatorServiceImpl implements ValidatorService {
return getValidationService().getJobSummary(repositoryService.getRepositoryInterface(repoId).stream().map(RepositoryInterface::getBaseUrl).collect(Collectors.toList()),limit);
}
}

82
src/main/java/eu/dnetlib/repo/manager/utils/CrisValidatorUtils.java Normal file
View File

@ -0,0 +1,82 @@
package eu.dnetlib.repo.manager.utils;
import eu.dnetlib.domain.functionality.validator.JobResultEntry;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import org.eurocris.openaire.cris.validator.model.Job;
import org.eurocris.openaire.cris.validator.model.RuleResults;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
public class CrisValidatorUtils {
private static final DateFormat DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private static final DateFormat DURATION_FORMATTER = new SimpleDateFormat("HH 'hours' mm 'min' ss 'sec'");
public static StoredJob convertJobToStoredJob(Job job) {
StoredJob sj = new StoredJob();
sj.setId(job.getId().hashCode());
sj.setValidationStatus(job.getStatus());
if (job.getDateFinished() != null) {
sj.setEnded(DATE_FORMATTER.format(job.getDateFinished()));
sj.setDuration(DURATION_FORMATTER.format(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime())));
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(DATE_FORMATTER.format(job.getDateStarted()));
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setValidationType("CU");
sj.setJobType("CRIS Validation");
sj.setGuidelinesShortName("For CRIS Managers 1.1");
sj.setValidationSet("-");
// Set status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getUsageJobStatus());
sj.setContentJobStatus(job.getContentJobStatus());
// Set scores
// sj.setFilteredScores(); // TODO what is this?
sj.setContentJobScore(job.getContentScore());
sj.setUsageJobScore(job.getUsageScore());
sj.setResultEntries(CrisValidatorUtils.crisResultsToJobEntries(job));
sj.setRecordsTested(job.getRecordsTested());
return sj;
}
public static List<JobResultEntry> crisResultsToJobEntries(Job crisJob) {
List<JobResultEntry> jobResultEntries = new ArrayList<>();
for (RuleResults ruleResults : crisJob.getRuleResults()) {
JobResultEntry jobResultEntry = new JobResultEntry();
jobResultEntry.setName(ruleResults.getRuleMethodName());
jobResultEntry.setRuleId(ruleResults.getRuleId());
jobResultEntry.setMandatory(true);
jobResultEntry.setWeight(Math.round(ruleResults.getWeight()));
jobResultEntry.setType(ruleResults.getType());
jobResultEntry.setHasErrors(false);
if (ruleResults.getCount() == 0) {
jobResultEntry.setSuccesses("-");
} else {
jobResultEntry.setSuccesses(ruleResults.getCount() - ruleResults.getFailed() + "/" + ruleResults.getCount());
}
if (ruleResults.getErrorMessages() != null && !ruleResults.getErrorMessages().isEmpty()) {
jobResultEntry.setErrors(ruleResults.getErrorMessages());
jobResultEntry.setHasErrors(true);
}
jobResultEntries.add(jobResultEntry);
}
return jobResultEntries;
}
private CrisValidatorUtils() {
}
}