a rough implementation for cris validation using openaire-cris-validator
This commit is contained in:
parent
c240fc1030
commit
0b512d17d5
14
pom.xml
14
pom.xml
|
@ -274,6 +274,18 @@
|
||||||
<version>1.1-rev-1</version>
|
<version>1.1-rev-1</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.eurocris</groupId>
|
||||||
|
<artifactId>openaire-cris-validator</artifactId>
|
||||||
|
<version>1.0.1-SNAPSHOT</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.mockito</groupId>
|
||||||
|
<artifactId>mockito-core</artifactId>
|
||||||
|
<version>2.26.0</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
|
@ -301,7 +313,7 @@
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>cz.habarta.typescript-generator</groupId>
|
<groupId>cz.habarta.typescript-generator</groupId>
|
||||||
<artifactId>typescript-generator-maven-plugin</artifactId>
|
<artifactId>typescript-generator-maven-plugin</artifactId>
|
||||||
<version>2.1.406</version>
|
<version>2.16.538</version>
|
||||||
<executions>
|
<executions>
|
||||||
<execution>
|
<execution>
|
||||||
<id>java to typeScript</id>
|
<id>java to typeScript</id>
|
||||||
|
|
|
@ -19,7 +19,9 @@ import javax.annotation.PostConstruct;
|
||||||
@EnableAspectJAutoProxy
|
@EnableAspectJAutoProxy
|
||||||
@EnableCircuitBreaker
|
@EnableCircuitBreaker
|
||||||
@PropertySource(value = {"classpath:application.properties"} )
|
@PropertySource(value = {"classpath:application.properties"} )
|
||||||
@ComponentScan(basePackages = "eu.dnetlib.repo.manager.*")
|
@ComponentScan(basePackages = {
|
||||||
|
"org.eurocris.openaire.cris.validator.service",
|
||||||
|
"eu.dnetlib.repo.manager.*"})
|
||||||
public class Config {
|
public class Config {
|
||||||
|
|
||||||
private static Logger LOGGER = Logger.getLogger(Config.class);
|
private static Logger LOGGER = Logger.getLogger(Config.class);
|
||||||
|
|
|
@ -2,19 +2,32 @@ package eu.dnetlib.repo.manager.service;
|
||||||
|
|
||||||
import eu.dnetlib.api.functionality.ValidatorService;
|
import eu.dnetlib.api.functionality.ValidatorService;
|
||||||
import eu.dnetlib.api.functionality.ValidatorServiceException;
|
import eu.dnetlib.api.functionality.ValidatorServiceException;
|
||||||
|
import eu.dnetlib.domain.functionality.validator.JobForValidation;
|
||||||
import eu.dnetlib.domain.functionality.validator.StoredJob;
|
import eu.dnetlib.domain.functionality.validator.StoredJob;
|
||||||
import eu.dnetlib.repo.manager.shared.Constants;
|
import eu.dnetlib.repo.manager.shared.Constants;
|
||||||
import eu.dnetlib.repo.manager.shared.JobsOfUser;
|
import eu.dnetlib.repo.manager.shared.JobsOfUser;
|
||||||
import gr.uoa.di.driver.util.ServiceLocator;
|
import gr.uoa.di.driver.util.ServiceLocator;
|
||||||
import org.apache.log4j.Logger;
|
import org.apache.log4j.Logger;
|
||||||
|
import org.eurocris.openaire.cris.validator.model.Job;
|
||||||
|
import org.eurocris.openaire.cris.validator.service.MapJobDao;
|
||||||
import org.json.JSONException;
|
import org.json.JSONException;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
|
import javax.mail.Store;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service("monitorService")
|
@Service("monitorService")
|
||||||
public class MonitorServiceImpl implements MonitorService {
|
public class MonitorServiceImpl implements MonitorService {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private MapJobDao crisJobs;
|
||||||
|
|
||||||
@Resource(name = "validatorServiceLocator")
|
@Resource(name = "validatorServiceLocator")
|
||||||
private ServiceLocator<ValidatorService> validatorServiceLocator;
|
private ServiceLocator<ValidatorService> validatorServiceLocator;
|
||||||
|
|
||||||
|
@ -49,6 +62,7 @@ public class MonitorServiceImpl implements MonitorService {
|
||||||
JobsOfUser retJobs = new JobsOfUser();
|
JobsOfUser retJobs = new JobsOfUser();
|
||||||
retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),
|
retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),
|
||||||
Integer.parseInt(limit), dateFrom, dateTo, validationStatus));
|
Integer.parseInt(limit), dateFrom, dateTo, validationStatus));
|
||||||
|
|
||||||
if (Boolean.parseBoolean(includeJobsTotal)) {
|
if (Boolean.parseBoolean(includeJobsTotal)) {
|
||||||
retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null));
|
retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null));
|
||||||
retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL));
|
retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL));
|
||||||
|
@ -73,7 +87,43 @@ public class MonitorServiceImpl implements MonitorService {
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
List<StoredJob> jobs = new ArrayList<>();
|
||||||
|
|
||||||
|
List<Job> cj = crisJobs.getJobs(user);
|
||||||
|
for (Job job : cj) {
|
||||||
|
|
||||||
|
StoredJob sj = converJobToStoredJob(job);
|
||||||
|
|
||||||
|
|
||||||
|
// filter out entries based on 'validationStatus'
|
||||||
|
if ("all".equals(validationStatus)) {
|
||||||
|
jobs.add(sj);
|
||||||
|
} else {
|
||||||
|
if (sj.getValidationStatus().equals(validationStatus)) {
|
||||||
|
jobs.add(sj);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// add to CRIS Jan jobs all other jobs
|
||||||
|
if (retJobs.getJobs() != null) {
|
||||||
|
jobs.addAll(retJobs.getJobs());
|
||||||
|
}
|
||||||
|
// set all jobs back to retJobs
|
||||||
|
retJobs.setJobs(jobs);
|
||||||
|
|
||||||
|
// fix number of jobs
|
||||||
|
if (Boolean.parseBoolean(includeJobsTotal)) {
|
||||||
|
retJobs.setTotalJobs(retJobs.getTotalJobs() + crisJobs.getJobs(user).size());
|
||||||
|
retJobs.setTotalJobsSuccessful(retJobs.getTotalJobsSuccessful() + crisJobs.getJobs(user, Job.Status.SUCCESSFUL.getKey()).size());
|
||||||
|
retJobs.setTotalJobsFailed(retJobs.getTotalJobsFailed() + crisJobs.getJobs(user, Job.Status.FAILED.getKey()).size());
|
||||||
|
retJobs.setTotalJobsOngoing(retJobs.getTotalJobsOngoing() + crisJobs.getJobs(user, Job.Status.ONGOING.getKey()).size());
|
||||||
|
}
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
return retJobs;
|
return retJobs;
|
||||||
|
|
||||||
|
@ -100,12 +150,58 @@ public class MonitorServiceImpl implements MonitorService {
|
||||||
public StoredJob getJobSummary(String jobId,
|
public StoredJob getJobSummary(String jobId,
|
||||||
String groupBy) throws JSONException {
|
String groupBy) throws JSONException {
|
||||||
LOGGER.debug("Getting job summary with id : " + jobId);
|
LOGGER.debug("Getting job summary with id : " + jobId);
|
||||||
|
StoredJob job = null;
|
||||||
try {
|
try {
|
||||||
return getValidationService().getStoredJob(Integer.parseInt(jobId), groupBy);
|
job = getValidationService().getStoredJob(Integer.parseInt(jobId), groupBy);
|
||||||
} catch (ValidatorServiceException e) {
|
} catch (ValidatorServiceException e) {
|
||||||
LOGGER.error(e);
|
LOGGER.error(e);
|
||||||
}
|
}
|
||||||
return null;
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
if (job == null) {
|
||||||
|
// not a good way to do it but Job id field is string
|
||||||
|
List<Job> cJobs = crisJobs.getAll().stream().filter(j -> j.getId().hashCode() == Integer.parseInt(jobId)).collect(Collectors.toList());
|
||||||
|
if (!cJobs.isEmpty()) {
|
||||||
|
job = converJobToStoredJob(cJobs.get(0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
private StoredJob converJobToStoredJob(Job job) {
|
||||||
|
StoredJob sj = new StoredJob();
|
||||||
|
sj.setId(job.getId().hashCode());
|
||||||
|
sj.setValidationStatus(job.getStatus());
|
||||||
|
if (job.getDateFinished() != null) {
|
||||||
|
sj.setEnded(job.getDateFinished().toString());
|
||||||
|
sj.setDuration(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()).toString());
|
||||||
|
} else {
|
||||||
|
sj.setEnded("-");
|
||||||
|
sj.setDuration("-");
|
||||||
|
}
|
||||||
|
sj.setStarted(job.getDateStarted().toString());
|
||||||
|
|
||||||
|
sj.setUserEmail(job.getUser());
|
||||||
|
sj.setCris(true);
|
||||||
|
sj.setBaseUrl(job.getUrl());
|
||||||
|
sj.setJobType("CRIS Validation");
|
||||||
|
|
||||||
|
// TODO: status
|
||||||
|
sj.setValidationStatus(job.getStatus());
|
||||||
|
sj.setUsageJobStatus(job.getStatus());
|
||||||
|
sj.setContentJobStatus(job.getStatus());
|
||||||
|
|
||||||
|
// TODO: scores
|
||||||
|
// sj.setFilteredScores();
|
||||||
|
sj.setContentJobScore(job.getScore());
|
||||||
|
sj.setUsageJobScore(job.getScore());
|
||||||
|
|
||||||
|
sj.setValidationType("CRIS Validation");
|
||||||
|
|
||||||
|
return sj;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,10 +3,7 @@ package eu.dnetlib.repo.manager.service;
|
||||||
import eu.dnetlib.api.functionality.ValidatorServiceException;
|
import eu.dnetlib.api.functionality.ValidatorServiceException;
|
||||||
import eu.dnetlib.domain.data.Repository;
|
import eu.dnetlib.domain.data.Repository;
|
||||||
import eu.dnetlib.domain.data.RepositoryInterface;
|
import eu.dnetlib.domain.data.RepositoryInterface;
|
||||||
import eu.dnetlib.domain.functionality.validator.JobForValidation;
|
import eu.dnetlib.domain.functionality.validator.*;
|
||||||
import eu.dnetlib.domain.functionality.validator.JobResultEntry;
|
|
||||||
import eu.dnetlib.domain.functionality.validator.RuleSet;
|
|
||||||
import eu.dnetlib.domain.functionality.validator.StoredJob;
|
|
||||||
import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;
|
import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;
|
||||||
import eu.dnetlib.repo.manager.shared.Constants;
|
import eu.dnetlib.repo.manager.shared.Constants;
|
||||||
import eu.dnetlib.repo.manager.shared.InterfaceInformation;
|
import eu.dnetlib.repo.manager.shared.InterfaceInformation;
|
||||||
|
@ -14,6 +11,9 @@ import eu.dnetlib.repo.manager.shared.ValidationServiceException;
|
||||||
import eu.dnetlib.repo.manager.utils.OaiTools;
|
import eu.dnetlib.repo.manager.utils.OaiTools;
|
||||||
import gr.uoa.di.driver.util.ServiceLocator;
|
import gr.uoa.di.driver.util.ServiceLocator;
|
||||||
import org.apache.log4j.Logger;
|
import org.apache.log4j.Logger;
|
||||||
|
import org.eurocris.openaire.cris.validator.model.Job;
|
||||||
|
import org.eurocris.openaire.cris.validator.service.JobExecutor;
|
||||||
|
import org.eurocris.openaire.cris.validator.service.MapJobDao;
|
||||||
import org.json.JSONException;
|
import org.json.JSONException;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.http.HttpStatus;
|
import org.springframework.http.HttpStatus;
|
||||||
|
@ -61,6 +61,12 @@ public class ValidatorServiceImpl implements ValidatorService {
|
||||||
@Autowired
|
@Autowired
|
||||||
private EmailUtils emailUtils;
|
private EmailUtils emailUtils;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private JobExecutor crisValidatorExecutor;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private MapJobDao crisJobs;
|
||||||
|
|
||||||
@PostConstruct
|
@PostConstruct
|
||||||
private void loadRules(){
|
private void loadRules(){
|
||||||
LOGGER.debug("PostConstruct method! Load rules!");
|
LOGGER.debug("PostConstruct method! Load rules!");
|
||||||
|
@ -84,6 +90,35 @@ public class ValidatorServiceImpl implements ValidatorService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
int ruleId = -1000;
|
||||||
|
Rule rule = new Rule();
|
||||||
|
rule.setType("cris");
|
||||||
|
rule.setName("CRIS Rules");
|
||||||
|
rule.setId(ruleId);
|
||||||
|
rule.setMandatory(true);
|
||||||
|
RuleSet crisRuleSet = new RuleSet();
|
||||||
|
crisRuleSet.setContentRulesIds(Collections.singleton(ruleId));
|
||||||
|
crisRuleSet.setContentRules(Collections.singletonList(rule));
|
||||||
|
crisRuleSet.setUsageRulesIds(Collections.singleton(ruleId));
|
||||||
|
crisRuleSet.setUsageRules(Collections.singletonList(rule));
|
||||||
|
crisRuleSet.setId(-1);
|
||||||
|
crisRuleSet.setDescription("Validates using the CRIS Validator implementation");
|
||||||
|
crisRuleSet.setName("CRIS Validation");
|
||||||
|
|
||||||
|
String crisKey = Constants.VALIDATION_MODE_CRIS;
|
||||||
|
if (rulesetMap.containsKey(crisKey))
|
||||||
|
rulesetMap.get(crisKey).add(crisRuleSet);
|
||||||
|
else {
|
||||||
|
List<RuleSet> ruleSets = new ArrayList<RuleSet>();
|
||||||
|
ruleSets.add(crisRuleSet);
|
||||||
|
rulesetMap.put(crisKey, ruleSets);
|
||||||
|
}
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
} catch (ValidatorServiceException e) {
|
} catch (ValidatorServiceException e) {
|
||||||
LOGGER.error(e);
|
LOGGER.error(e);
|
||||||
}
|
}
|
||||||
|
@ -96,7 +131,18 @@ public class ValidatorServiceImpl implements ValidatorService {
|
||||||
LOGGER.debug("Submit job for validation with id : " + jobForValidation.getDatasourceId());
|
LOGGER.debug("Submit job for validation with id : " + jobForValidation.getDatasourceId());
|
||||||
try {
|
try {
|
||||||
emailUtils.sendSubmitJobForValidationEmail(SecurityContextHolder.getContext().getAuthentication(),jobForValidation);
|
emailUtils.sendSubmitJobForValidationEmail(SecurityContextHolder.getContext().getAuthentication(),jobForValidation);
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
if (jobForValidation.getSelectedContentRules().size() == 1 &&
|
||||||
|
jobForValidation.getSelectedContentRules().contains(-1000)) {
|
||||||
|
crisValidatorExecutor.submit(jobForValidation.getBaseUrl(), jobForValidation.getUserEmail());
|
||||||
|
} else {
|
||||||
this.getValidationService().submitValidationJob(jobForValidation);
|
this.getValidationService().submitValidationJob(jobForValidation);
|
||||||
|
}
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// this.getValidationService().submitValidationJob(jobForValidation);
|
||||||
} catch (ValidatorServiceException e) {
|
} catch (ValidatorServiceException e) {
|
||||||
LOGGER.debug("Exception on submitJobForValidation" , e);
|
LOGGER.debug("Exception on submitJobForValidation" , e);
|
||||||
emailUtils.reportException(e);
|
emailUtils.reportException(e);
|
||||||
|
@ -199,7 +245,47 @@ public class ValidatorServiceImpl implements ValidatorService {
|
||||||
String dateFrom,
|
String dateFrom,
|
||||||
String dateTo,
|
String dateTo,
|
||||||
String validationStatus ) throws ValidatorServiceException {
|
String validationStatus ) throws ValidatorServiceException {
|
||||||
return getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus);
|
// return getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus);
|
||||||
|
List<StoredJob> jobs = getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus);
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
if (jobs == null) {
|
||||||
|
jobs = new ArrayList<>();
|
||||||
|
}
|
||||||
|
List<Job> cj = crisJobs.getJobs(user);
|
||||||
|
for (Job job : cj) {
|
||||||
|
StoredJob sj = new StoredJob();
|
||||||
|
|
||||||
|
if (job.getDateFinished() != null) {
|
||||||
|
sj.setEnded(job.getDateFinished().toString());
|
||||||
|
sj.setDuration(Long.toString(job.getDateFinished().getTime() - job.getDateStarted().getTime()) + " seconds");
|
||||||
|
} else {
|
||||||
|
sj.setEnded("-");
|
||||||
|
sj.setDuration("-");
|
||||||
|
}
|
||||||
|
sj.setStarted(job.getDateStarted().toString());
|
||||||
|
|
||||||
|
sj.setUserEmail(job.getUser());
|
||||||
|
sj.setCris(true);
|
||||||
|
sj.setBaseUrl(job.getUrl());
|
||||||
|
sj.setJobType("CRIS Validation");
|
||||||
|
|
||||||
|
// TODO: status
|
||||||
|
sj.setValidationStatus(job.getStatus());
|
||||||
|
sj.setUsageJobStatus(job.getStatus());
|
||||||
|
sj.setContentJobStatus(job.getStatus());
|
||||||
|
|
||||||
|
// TODO: scores
|
||||||
|
// sj.setFilteredScores();
|
||||||
|
sj.setContentJobScore(job.getScore());
|
||||||
|
sj.setUsageJobScore(job.getScore());
|
||||||
|
|
||||||
|
jobs.add(sj);
|
||||||
|
}
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
/////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
return jobs;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
infrastructure.name=integration
|
infrastructure.name=integration
|
||||||
|
|
||||||
## Container properties
|
## Container properties
|
||||||
container.hostname=estella.athenarc.gr
|
container.hostname=localhost
|
||||||
container.context=uoa-repository-manager-service
|
container.context=uoa-repository-manager-service
|
||||||
container.port=8480
|
container.port=8480
|
||||||
|
|
||||||
|
@ -70,7 +70,7 @@ webapp.dev.front = http://localhost:4200/landing
|
||||||
|
|
||||||
##REDIS-AAI
|
##REDIS-AAI
|
||||||
|
|
||||||
redis.host=estella.athenarc.gr
|
redis.host=localhost
|
||||||
redis.port=6379
|
redis.port=6379
|
||||||
redis.password=redis123
|
redis.password=redis123
|
||||||
aai.mode=develop
|
aai.mode=develop
|
||||||
|
|
Loading…
Reference in New Issue