MaDgIK
/
uoa-repository-manager-service
13
2
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

a rough implementation for cris validation using openaire-cris-validator

This commit is contained in:
Konstantinos Spyrou 2019-11-11 14:35:57 +00:00
parent c240fc1030
commit 0b512d17d5
5 changed files with 286 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pom.xml
View File

@ -274,6 +274,18 @@
<version>1.1-rev-1</version>
</dependency>
<dependency>
<groupId>org.eurocris</groupId>
<artifactId>openaire-cris-validator</artifactId>
<version>1.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>2.26.0</version>
<scope>test</scope>
</dependency>
</dependencies>
@ -301,7 +313,7 @@
<plugin>
<groupId>cz.habarta.typescript-generator</groupId>
<artifactId>typescript-generator-maven-plugin</artifactId>
<version>2.1.406</version>
<version>2.16.538</version>
<executions>
<execution>
<id>java to typeScript</id>

4
src/main/java/eu/dnetlib/repo/manager/config/Config.java
View File

@ -19,7 +19,9 @@ import javax.annotation.PostConstruct;
@EnableAspectJAutoProxy
@EnableCircuitBreaker
@PropertySource(value = {"classpath:application.properties"} )
@ComponentScan(basePackages = "eu.dnetlib.repo.manager.*")
@ComponentScan(basePackages = {
"org.eurocris.openaire.cris.validator.service",
"eu.dnetlib.repo.manager.*"})
public class Config {
private static Logger LOGGER = Logger.getLogger(Config.class);

100
src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java
View File

@ -2,19 +2,32 @@ package eu.dnetlib.repo.manager.service;
import eu.dnetlib.api.functionality.ValidatorService;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.functionality.validator.JobForValidation;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import eu.dnetlib.repo.manager.shared.Constants;
import eu.dnetlib.repo.manager.shared.JobsOfUser;
import gr.uoa.di.driver.util.ServiceLocator;
import org.apache.log4j.Logger;
import org.eurocris.openaire.cris.validator.model.Job;
import org.eurocris.openaire.cris.validator.service.MapJobDao;
import org.json.JSONException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import javax.mail.Store;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
@Service("monitorService")
public class MonitorServiceImpl implements MonitorService {
@Autowired
private MapJobDao crisJobs;
@Resource(name = "validatorServiceLocator")
private ServiceLocator<ValidatorService> validatorServiceLocator;
@ -49,6 +62,7 @@ public class MonitorServiceImpl implements MonitorService {
JobsOfUser retJobs = new JobsOfUser();
retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),
Integer.parseInt(limit), dateFrom, dateTo, validationStatus));
if (Boolean.parseBoolean(includeJobsTotal)) {
retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null));
retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL));
@ -73,7 +87,43 @@ public class MonitorServiceImpl implements MonitorService {
}
}
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
List<StoredJob> jobs = new ArrayList<>();
List<Job> cj = crisJobs.getJobs(user);
for (Job job : cj) {
StoredJob sj = converJobToStoredJob(job);
// filter out entries based on 'validationStatus'
if ("all".equals(validationStatus)) {
jobs.add(sj);
} else {
if (sj.getValidationStatus().equals(validationStatus)) {
jobs.add(sj);
}
}
}
// add to CRIS Jan jobs all other jobs
if (retJobs.getJobs() != null) {
jobs.addAll(retJobs.getJobs());
}
// set all jobs back to retJobs
retJobs.setJobs(jobs);
// fix number of jobs
if (Boolean.parseBoolean(includeJobsTotal)) {
retJobs.setTotalJobs(retJobs.getTotalJobs() + crisJobs.getJobs(user).size());
retJobs.setTotalJobsSuccessful(retJobs.getTotalJobsSuccessful() + crisJobs.getJobs(user, Job.Status.SUCCESSFUL.getKey()).size());
retJobs.setTotalJobsFailed(retJobs.getTotalJobsFailed() + crisJobs.getJobs(user, Job.Status.FAILED.getKey()).size());
retJobs.setTotalJobsOngoing(retJobs.getTotalJobsOngoing() + crisJobs.getJobs(user, Job.Status.ONGOING.getKey()).size());
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
return retJobs;
@ -100,12 +150,58 @@ public class MonitorServiceImpl implements MonitorService {
public StoredJob getJobSummary(String jobId,
String groupBy) throws JSONException {
LOGGER.debug("Getting job summary with id : " + jobId);
StoredJob job = null;
try {
return getValidationService().getStoredJob(Integer.parseInt(jobId), groupBy);
job = getValidationService().getStoredJob(Integer.parseInt(jobId), groupBy);
} catch (ValidatorServiceException e) {
LOGGER.error(e);
}
return null;
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
if (job == null) {
// not a good way to do it but Job id field is string
List<Job> cJobs = crisJobs.getAll().stream().filter(j -> j.getId().hashCode() == Integer.parseInt(jobId)).collect(Collectors.toList());
if (!cJobs.isEmpty()) {
job = converJobToStoredJob(cJobs.get(0));
}
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
return job;
}
private StoredJob converJobToStoredJob(Job job) {
StoredJob sj = new StoredJob();
sj.setId(job.getId().hashCode());
sj.setValidationStatus(job.getStatus());
if (job.getDateFinished() != null) {
sj.setEnded(job.getDateFinished().toString());
sj.setDuration(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()).toString());
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(job.getDateStarted().toString());
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setJobType("CRIS Validation");
// TODO: status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getStatus());
sj.setContentJobStatus(job.getStatus());
// TODO: scores
// sj.setFilteredScores();
sj.setContentJobScore(job.getScore());
sj.setUsageJobScore(job.getScore());
sj.setValidationType("CRIS Validation");
return sj;
}
}

96
src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java
View File

@ -3,10 +3,7 @@ package eu.dnetlib.repo.manager.service;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.data.Repository;
import eu.dnetlib.domain.data.RepositoryInterface;
import eu.dnetlib.domain.functionality.validator.JobForValidation;
import eu.dnetlib.domain.functionality.validator.JobResultEntry;
import eu.dnetlib.domain.functionality.validator.RuleSet;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import eu.dnetlib.domain.functionality.validator.*;
import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;
import eu.dnetlib.repo.manager.shared.Constants;
import eu.dnetlib.repo.manager.shared.InterfaceInformation;
@ -14,6 +11,9 @@ import eu.dnetlib.repo.manager.shared.ValidationServiceException;
import eu.dnetlib.repo.manager.utils.OaiTools;
import gr.uoa.di.driver.util.ServiceLocator;
import org.apache.log4j.Logger;
import org.eurocris.openaire.cris.validator.model.Job;
import org.eurocris.openaire.cris.validator.service.JobExecutor;
import org.eurocris.openaire.cris.validator.service.MapJobDao;
import org.json.JSONException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
@ -61,6 +61,12 @@ public class ValidatorServiceImpl implements ValidatorService {
@Autowired
private EmailUtils emailUtils;
@Autowired
private JobExecutor crisValidatorExecutor;
@Autowired
private MapJobDao crisJobs;
@PostConstruct
private void loadRules(){
LOGGER.debug("PostConstruct method! Load rules!");
@ -84,6 +90,35 @@ public class ValidatorServiceImpl implements ValidatorService {
}
}
}
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
int ruleId = -1000;
Rule rule = new Rule();
rule.setType("cris");
rule.setName("CRIS Rules");
rule.setId(ruleId);
rule.setMandatory(true);
RuleSet crisRuleSet = new RuleSet();
crisRuleSet.setContentRulesIds(Collections.singleton(ruleId));
crisRuleSet.setContentRules(Collections.singletonList(rule));
crisRuleSet.setUsageRulesIds(Collections.singleton(ruleId));
crisRuleSet.setUsageRules(Collections.singletonList(rule));
crisRuleSet.setId(-1);
crisRuleSet.setDescription("Validates using the CRIS Validator implementation");
crisRuleSet.setName("CRIS Validation");
String crisKey = Constants.VALIDATION_MODE_CRIS;
if (rulesetMap.containsKey(crisKey))
rulesetMap.get(crisKey).add(crisRuleSet);
else {
List<RuleSet> ruleSets = new ArrayList<RuleSet>();
ruleSets.add(crisRuleSet);
rulesetMap.put(crisKey, ruleSets);
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
} catch (ValidatorServiceException e) {
LOGGER.error(e);
}
@ -96,7 +131,18 @@ public class ValidatorServiceImpl implements ValidatorService {
LOGGER.debug("Submit job for validation with id : " + jobForValidation.getDatasourceId());
try {
emailUtils.sendSubmitJobForValidationEmail(SecurityContextHolder.getContext().getAuthentication(),jobForValidation);
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
if (jobForValidation.getSelectedContentRules().size() == 1 &&
jobForValidation.getSelectedContentRules().contains(-1000)) {
crisValidatorExecutor.submit(jobForValidation.getBaseUrl(), jobForValidation.getUserEmail());
} else {
this.getValidationService().submitValidationJob(jobForValidation);
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
// this.getValidationService().submitValidationJob(jobForValidation);
} catch (ValidatorServiceException e) {
LOGGER.debug("Exception on submitJobForValidation" , e);
emailUtils.reportException(e);
@ -199,7 +245,47 @@ public class ValidatorServiceImpl implements ValidatorService {
String dateFrom,
String dateTo,
String validationStatus ) throws ValidatorServiceException {
return getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus);
// return getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus);
List<StoredJob> jobs = getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus);
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
if (jobs == null) {
jobs = new ArrayList<>();
}
List<Job> cj = crisJobs.getJobs(user);
for (Job job : cj) {
StoredJob sj = new StoredJob();
if (job.getDateFinished() != null) {
sj.setEnded(job.getDateFinished().toString());
sj.setDuration(Long.toString(job.getDateFinished().getTime() - job.getDateStarted().getTime()) + " seconds");
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(job.getDateStarted().toString());
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setJobType("CRIS Validation");
// TODO: status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getStatus());
sj.setContentJobStatus(job.getStatus());
// TODO: scores
// sj.setFilteredScores();
sj.setContentJobScore(job.getScore());
sj.setUsageJobScore(job.getScore());
jobs.add(sj);
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
return jobs;
}
@Override

4
src/main/resources/application.properties
View File

@ -1,7 +1,7 @@
infrastructure.name=integration
## Container properties
container.hostname=estella.athenarc.gr
container.hostname=localhost
container.context=uoa-repository-manager-service
container.port=8480
@ -70,7 +70,7 @@ webapp.dev.front = http://localhost:4200/landing
##REDIS-AAI
redis.host=estella.athenarc.gr
redis.host=localhost
redis.port=6379
redis.password=redis123
aai.mode=develop