merged branch cris r57561-HEAD

This commit is contained in:
Antonis Lempesis 2019-11-20 13:03:40 +00:00
parent df23be95ae
commit 1fbfaa454f
5 changed files with 286 additions and 90 deletions

82
pom.xml
View File

@ -240,39 +240,51 @@
<version>1.1.5.RELEASE</version>
</dependency>
<!-- https://mvnrepository.com/artifact/javax.xml.ws/jaxws-api -->
<dependency>
<groupId>javax.xml.ws</groupId>
<artifactId>jaxws-api</artifactId>
<version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/javax.jws/javax.jws-api -->
<dependency>
<groupId>javax.jws</groupId>
<artifactId>javax.jws-api</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
<version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.sun.xml.bind/jaxb-core -->
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-core</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
<version>1.1-rev-1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/javax.xml.ws/jaxws-api -->
<dependency>
<groupId>javax.xml.ws</groupId>
<artifactId>jaxws-api</artifactId>
<version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/javax.jws/javax.jws-api -->
<dependency>
<groupId>javax.jws</groupId>
<artifactId>javax.jws-api</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
<version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.sun.xml.bind/jaxb-core -->
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-core</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
<version>1.1-rev-1</version>
</dependency>
<dependency>
<groupId>org.eurocris</groupId>
<artifactId>openaire-cris-validator</artifactId>
<version>1.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>2.26.0</version>
<scope>test</scope>
</dependency>
</dependencies>
@ -280,7 +292,7 @@
<build>
<outputDirectory>${project.build.directory}/${project.build.finalName}/WEB-INF/classes</outputDirectory>
<plugins>
<plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
@ -301,7 +313,7 @@
<plugin>
<groupId>cz.habarta.typescript-generator</groupId>
<artifactId>typescript-generator-maven-plugin</artifactId>
<version>2.1.406</version>
<version>2.16.538</version>
<executions>
<execution>
<id>java to typeScript</id>

View File

@ -19,7 +19,9 @@ import javax.annotation.PostConstruct;
@EnableAspectJAutoProxy
@EnableCircuitBreaker
@PropertySource(value = {"classpath:application.properties"} )
@ComponentScan(basePackages = "eu.dnetlib.repo.manager.*")
@ComponentScan(basePackages = {
"org.eurocris.openaire.cris.validator.service",
"eu.dnetlib.repo.manager.*"})
public class Config {
private static Logger LOGGER = Logger.getLogger(Config.class);

View File

@ -2,19 +2,32 @@ package eu.dnetlib.repo.manager.service;
import eu.dnetlib.api.functionality.ValidatorService;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.functionality.validator.JobForValidation;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import eu.dnetlib.repo.manager.shared.Constants;
import eu.dnetlib.repo.manager.shared.JobsOfUser;
import gr.uoa.di.driver.util.ServiceLocator;
import org.apache.log4j.Logger;
import org.eurocris.openaire.cris.validator.model.Job;
import org.eurocris.openaire.cris.validator.service.MapJobDao;
import org.json.JSONException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import javax.mail.Store;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
@Service("monitorService")
public class MonitorServiceImpl implements MonitorService {
@Autowired
private MapJobDao crisJobs;
@Resource(name = "validatorServiceLocator")
private ServiceLocator<ValidatorService> validatorServiceLocator;
@ -49,6 +62,7 @@ public class MonitorServiceImpl implements MonitorService {
JobsOfUser retJobs = new JobsOfUser();
retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),
Integer.parseInt(limit), dateFrom, dateTo, validationStatus));
if (Boolean.parseBoolean(includeJobsTotal)) {
retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null));
retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL));
@ -73,7 +87,43 @@ public class MonitorServiceImpl implements MonitorService {
}
}
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
List<StoredJob> jobs = new ArrayList<>();
List<Job> cj = crisJobs.getJobs(user);
for (Job job : cj) {
StoredJob sj = converJobToStoredJob(job);
// filter out entries based on 'validationStatus'
if ("all".equals(validationStatus)) {
jobs.add(sj);
} else {
if (sj.getValidationStatus().equals(validationStatus)) {
jobs.add(sj);
}
}
}
// add to CRIS Jan jobs all other jobs
if (retJobs.getJobs() != null) {
jobs.addAll(retJobs.getJobs());
}
// set all jobs back to retJobs
retJobs.setJobs(jobs);
// fix number of jobs
if (Boolean.parseBoolean(includeJobsTotal)) {
retJobs.setTotalJobs(retJobs.getTotalJobs() + crisJobs.getJobs(user).size());
retJobs.setTotalJobsSuccessful(retJobs.getTotalJobsSuccessful() + crisJobs.getJobs(user, Job.Status.SUCCESSFUL.getKey()).size());
retJobs.setTotalJobsFailed(retJobs.getTotalJobsFailed() + crisJobs.getJobs(user, Job.Status.FAILED.getKey()).size());
retJobs.setTotalJobsOngoing(retJobs.getTotalJobsOngoing() + crisJobs.getJobs(user, Job.Status.ONGOING.getKey()).size());
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
return retJobs;
@ -100,12 +150,58 @@ public class MonitorServiceImpl implements MonitorService {
public StoredJob getJobSummary(String jobId,
String groupBy) throws JSONException {
LOGGER.debug("Getting job summary with id : " + jobId);
StoredJob job = null;
try {
return getValidationService().getStoredJob(Integer.parseInt(jobId), groupBy);
job = getValidationService().getStoredJob(Integer.parseInt(jobId), groupBy);
} catch (ValidatorServiceException e) {
LOGGER.error(e);
}
return null;
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
if (job == null) {
// not a good way to do it but Job id field is string
List<Job> cJobs = crisJobs.getAll().stream().filter(j -> j.getId().hashCode() == Integer.parseInt(jobId)).collect(Collectors.toList());
if (!cJobs.isEmpty()) {
job = converJobToStoredJob(cJobs.get(0));
}
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
return job;
}
private StoredJob converJobToStoredJob(Job job) {
StoredJob sj = new StoredJob();
sj.setId(job.getId().hashCode());
sj.setValidationStatus(job.getStatus());
if (job.getDateFinished() != null) {
sj.setEnded(job.getDateFinished().toString());
sj.setDuration(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()).toString());
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(job.getDateStarted().toString());
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setJobType("CRIS Validation");
// TODO: status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getStatus());
sj.setContentJobStatus(job.getStatus());
// TODO: scores
// sj.setFilteredScores();
sj.setContentJobScore(job.getScore());
sj.setUsageJobScore(job.getScore());
sj.setValidationType("CRIS Validation");
return sj;
}
}

View File

@ -3,10 +3,7 @@ package eu.dnetlib.repo.manager.service;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.data.Repository;
import eu.dnetlib.domain.data.RepositoryInterface;
import eu.dnetlib.domain.functionality.validator.JobForValidation;
import eu.dnetlib.domain.functionality.validator.JobResultEntry;
import eu.dnetlib.domain.functionality.validator.RuleSet;
import eu.dnetlib.domain.functionality.validator.StoredJob;
import eu.dnetlib.domain.functionality.validator.*;
import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;
import eu.dnetlib.repo.manager.shared.Constants;
import eu.dnetlib.repo.manager.shared.InterfaceInformation;
@ -14,6 +11,9 @@ import eu.dnetlib.repo.manager.shared.ValidationServiceException;
import eu.dnetlib.repo.manager.utils.OaiTools;
import gr.uoa.di.driver.util.ServiceLocator;
import org.apache.log4j.Logger;
import org.eurocris.openaire.cris.validator.model.Job;
import org.eurocris.openaire.cris.validator.service.JobExecutor;
import org.eurocris.openaire.cris.validator.service.MapJobDao;
import org.json.JSONException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
@ -61,6 +61,12 @@ public class ValidatorServiceImpl implements ValidatorService {
@Autowired
private EmailUtils emailUtils;
@Autowired
private JobExecutor crisValidatorExecutor;
@Autowired
private MapJobDao crisJobs;
@PostConstruct
private void loadRules(){
LOGGER.debug("PostConstruct method! Load rules!");
@ -84,6 +90,35 @@ public class ValidatorServiceImpl implements ValidatorService {
}
}
}
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
int ruleId = -1000;
Rule rule = new Rule();
rule.setType("cris");
rule.setName("CRIS Rules");
rule.setId(ruleId);
rule.setMandatory(true);
RuleSet crisRuleSet = new RuleSet();
crisRuleSet.setContentRulesIds(Collections.singleton(ruleId));
crisRuleSet.setContentRules(Collections.singletonList(rule));
crisRuleSet.setUsageRulesIds(Collections.singleton(ruleId));
crisRuleSet.setUsageRules(Collections.singletonList(rule));
crisRuleSet.setId(-1);
crisRuleSet.setDescription("Validates using the CRIS Validator implementation");
crisRuleSet.setName("CRIS Validation");
String crisKey = Constants.VALIDATION_MODE_CRIS;
if (rulesetMap.containsKey(crisKey))
rulesetMap.get(crisKey).add(crisRuleSet);
else {
List<RuleSet> ruleSets = new ArrayList<RuleSet>();
ruleSets.add(crisRuleSet);
rulesetMap.put(crisKey, ruleSets);
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
} catch (ValidatorServiceException e) {
LOGGER.error(e);
}
@ -96,7 +131,18 @@ public class ValidatorServiceImpl implements ValidatorService {
LOGGER.debug("Submit job for validation with id : " + jobForValidation.getDatasourceId());
try {
emailUtils.sendSubmitJobForValidationEmail(SecurityContextHolder.getContext().getAuthentication(),jobForValidation);
this.getValidationService().submitValidationJob(jobForValidation);
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
if (jobForValidation.getSelectedContentRules().size() == 1 &&
jobForValidation.getSelectedContentRules().contains(-1000)) {
crisValidatorExecutor.submit(jobForValidation.getBaseUrl(), jobForValidation.getUserEmail());
} else {
this.getValidationService().submitValidationJob(jobForValidation);
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
// this.getValidationService().submitValidationJob(jobForValidation);
} catch (ValidatorServiceException e) {
LOGGER.debug("Exception on submitJobForValidation" , e);
emailUtils.reportException(e);
@ -199,7 +245,47 @@ public class ValidatorServiceImpl implements ValidatorService {
String dateFrom,
String dateTo,
String validationStatus ) throws ValidatorServiceException {
return getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus);
// return getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus);
List<StoredJob> jobs = getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), Integer.parseInt(limit), dateFrom, dateTo, validationStatus);
/////////////////////////////////////////////////////////////////////////////////////////
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
/////////////////////////////////////////////////////////////////////////////////////////
if (jobs == null) {
jobs = new ArrayList<>();
}
List<Job> cj = crisJobs.getJobs(user);
for (Job job : cj) {
StoredJob sj = new StoredJob();
if (job.getDateFinished() != null) {
sj.setEnded(job.getDateFinished().toString());
sj.setDuration(Long.toString(job.getDateFinished().getTime() - job.getDateStarted().getTime()) + " seconds");
} else {
sj.setEnded("-");
sj.setDuration("-");
}
sj.setStarted(job.getDateStarted().toString());
sj.setUserEmail(job.getUser());
sj.setCris(true);
sj.setBaseUrl(job.getUrl());
sj.setJobType("CRIS Validation");
// TODO: status
sj.setValidationStatus(job.getStatus());
sj.setUsageJobStatus(job.getStatus());
sj.setContentJobStatus(job.getStatus());
// TODO: scores
// sj.setFilteredScores();
sj.setContentJobScore(job.getScore());
sj.setUsageJobScore(job.getScore());
jobs.add(sj);
}
/////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////
return jobs;
}
@Override

View File

@ -1,14 +1,14 @@
infrastructure.name=integration
## Container properties
container.hostname=estella.athenarc.gr
container.hostname=localhost
container.context=uoa-repository-manager-service
container.port=8480
IS.url = https://dev-openaire.d4science.org/is/services
ISLookUpService.url = ${IS.url}/isLookUp
ISRegistryService.url = ${IS.url}/isRegistry
ISSNService.url = ${IS.url}/services/isSN
IS.url=https://dev-openaire.d4science.org/is/services
ISLookUpService.url=${IS.url}/isLookUp
ISRegistryService.url=${IS.url}/isRegistry
ISSNService.url=${IS.url}/services/isSN
#ValidatorService.url=http://88.197.53.69:8080/validator-service/services/validatorWebService
#ValidatorService.url=http://beta.services.openaire.eu/validator-service/services/validatorWebService
@ -17,71 +17,71 @@ ValidatorService.url=http://localhost:8080/validator-service/services/validatorW
services.provide.adminEmail=provide-admin@openaire.eu
## Broker Service
services.broker.url = http://broker1-dev-dnet.d4science.org
services.broker.port = 8080
services.broker.api = api/
services.broker.openaire = openaireBroker
services.broker.url=http://broker1-dev-dnet.d4science.org
services.broker.port=8080
services.broker.api=api/
services.broker.openaire=openaireBroker
#api.baseAddress=https://dev-openaire.d4science.org/openaire
api.baseAddress=http://beta.services.openaire.eu/openaire
service.repository-rest-api.baseUrl =http://${container.hostname}:${container.port}
services.repo-manager.baseUrl = http://${container.hostname}:${container.port}/${container.context}
service.repository-rest-api.baseUrl=http://${container.hostname}:${container.port}
services.repo-manager.baseUrl=http://${container.hostname}:${container.port}/${container.context}
transport.soap.baseAddress = http://${container.hostname}:${container.port}/${container.context}
transport.soap.force.local.address = false
transport.soap.baseAddress=http://${container.hostname}:${container.port}/${container.context}
transport.soap.force.local.address=false
validator.results.url=https://${container.hostname}:${container.port}/compatibility/browseHistory/
services.repo-manager.adminEmail = antleb@di.uoa.gr
services.repo-manager.repository.testing.mode = false
services.repo-manager.deploy.environment = development
services.validator.mail.host = smtp.gmail.com
services.validator.mail.port = 465
services.validator.mail.fromAddress = no-reply@openaire.eu
services.validator.mail.replyToAddress = no-reply@openaire.eu
services.validator.mail.username = test.openaire@gmail.com
services.validator.mail.password = ^($*@$)*!$
services.validator.mail.authenticate = true
services.validator.mail.override = false
services.validator.mail.logonly = false
services.validator.mail.mode = ssl
services.validator.mail.debug = false
services.validator.mail.overrideEmail = antleb@di.uoa.gr
services.validator.mail.specialRecipients = antleb@di.uoa.gr
services.repo-manager.adminEmail=antleb@di.uoa.gr
services.repo-manager.repository.testing.mode=false
services.repo-manager.deploy.environment=development
services.validator.mail.host=smtp.gmail.com
services.validator.mail.port=465
services.validator.mail.fromAddress=no-reply@openaire.eu
services.validator.mail.replyToAddress=no-reply@openaire.eu
services.validator.mail.username=test.openaire@gmail.com
services.validator.mail.password=^($*@$)*!$
services.validator.mail.authenticate=true
services.validator.mail.override=false
services.validator.mail.logonly=false
services.validator.mail.mode=ssl
services.validator.mail.debug=false
services.validator.mail.overrideEmail=antleb@di.uoa.gr
services.validator.mail.specialRecipients=antleb@di.uoa.gr
services.validator.repoRegistration.override = antleb@di.uoa.gr
services.validator.repoRegistration.override=antleb@di.uoa.gr
repomanager.db.driverClassName = org.postgresql.Driver
repomanager.db.url = jdbc:postgresql://194.177.192.119:5432/repomanager
repomanager.db.username = dnet
repomanager.db.password = dnetPwd
repomanager.db.driverClassName=org.postgresql.Driver
repomanager.db.url=jdbc:postgresql://194.177.192.119:5432/repomanager
repomanager.db.username=dnet
repomanager.db.password=dnetPwd
services.repomanager.analyticsURL = http://analytics.openaire.eu/addsite.php?
services.repomanager.analyticsURL=http://analytics.openaire.eu/addsite.php?
topic_types.url = https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:topic_types.json
topic_types.url=https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:topic_types.json
oidc.issuer = https://aai.openaire.eu/oidc/
oidc.id = 767422b9-5461-4807-a80a-f9a2072d3a7d
oidc.secret = AMQtGlbTXNjwjhF0st28LmM6V0XypMdaVS7tJmGuYFlmH36iIv4t7tVqYuLYrNPkhnZ_GPUJvhymBhFupdgb6aU
oidc.issuer=https://aai.openaire.eu/oidc/
oidc.id=767422b9-5461-4807-a80a-f9a2072d3a7d
oidc.secret=AMQtGlbTXNjwjhF0st28LmM6V0XypMdaVS7tJmGuYFlmH36iIv4t7tVqYuLYrNPkhnZ_GPUJvhymBhFupdgb6aU
oidc.dev.home = http://localhost:8480/uoa-repository-manager-service/openid_connect_login
webapp.dev.front = http://localhost:4200/landing
oidc.dev.home=http://localhost:8480/uoa-repository-manager-service/openid_connect_login
webapp.dev.front=http://localhost:4200/landing
##REDIS-AAI
redis.host=estella.athenarc.gr
redis.host=localhost
redis.port=6379
redis.password=redis123
aai.mode=develop
services.repomanager.usageStatisticsDiagramsBaseURL = https://beta.openaire.eu/stats3/
services.repomanager.usageStatisticsNumbersBaseURL = https://beta.services.openaire.eu/usagestats/datasources/
services.repomanager.usagestats.adminEmail = antleb@di.uoa.gr
services.repomanager.usageStatisticsDiagramsBaseURL=https://beta.openaire.eu/stats3/
services.repomanager.usageStatisticsNumbersBaseURL=https://beta.services.openaire.eu/usagestats/datasources/
services.repomanager.usagestats.adminEmail=antleb@di.uoa.gr
##SUSHI
services.repomanager.usagestats.sushiliteEndpoint = http://beta.services.openaire.eu/usagestats/sushilite/
services.repomanager.usagestats.sushiliteEndpoint=http://beta.services.openaire.eu/usagestats/sushilite/
##SEARCH API
search.api.baseAddress=https://beta.services.openaire.eu/search/v2/api