progress with error and warning reports. not yet final. need to clean code and finalize the needed methods
This commit is contained in:
Katerina
parent
4d7836b4d7
commit
a15670b741
|
|
@ -3,12 +3,10 @@ package eu.dnetlib.validatorapi.controllers;
|
|||
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
||||
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
||||
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
||||
import eu.dnetlib.validatorapi.entities.RuleInfo;
|
||||
import eu.dnetlib.validatorapi.entities.SummaryResult;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJobResult;
|
||||
import eu.dnetlib.validatorapi.repository.ValidationJobRepository;
|
||||
import eu.dnetlib.validatorapi.repository.ValidationResultRepository;
|
||||
import eu.dnetlib.validatorapi.entities.*;
|
||||
import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository;
|
||||
import eu.dnetlib.validatorapi.repositories.ValidationJobRepository;
|
||||
import eu.dnetlib.validatorapi.repositories.ValidationResultRepository;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
|
@ -45,30 +43,33 @@ public class ValidationController {
|
|||
private final Logger log = LogManager.getLogger(this.getClass());
|
||||
private final ValidationJobRepository validationJobRepository;
|
||||
private final ValidationResultRepository validationResultRepository;
|
||||
private final ValidationIssueRepository validationIssueRepository;
|
||||
|
||||
@Autowired
|
||||
public ValidationController(ValidationJobRepository validationJobRepository, ValidationResultRepository validationResultRepository) {
|
||||
public ValidationController(ValidationJobRepository validationJobRepository, ValidationResultRepository validationResultRepository, ValidationIssueRepository validationIssueRepository) {
|
||||
this.validationJobRepository = validationJobRepository;
|
||||
this.validationResultRepository = validationResultRepository;
|
||||
this.validationIssueRepository = validationIssueRepository;
|
||||
}
|
||||
|
||||
@RequestMapping(value = {"/realValidator"}, method = RequestMethod.POST)
|
||||
public void validateRealOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
||||
@RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now
|
||||
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords //not in use now
|
||||
) {
|
||||
@RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now
|
||||
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
|
||||
@RequestParam(name="set", required = false) String set, @RequestParam(name="metadataPrefix", defaultValue = "oai_dc") String metadataPrefix,
|
||||
@RequestParam(name="batchsize", defaultValue = "50") int batchSize) {
|
||||
|
||||
ValidationJob validationJob = new ValidationJob(baseUrl, numberOfRecords);
|
||||
AbstractOpenAireProfile profile = initializeOpenAireProfile(guidelinesProfileName);
|
||||
AbstractOpenAireProfile fairProfile = initializeFairProfile(guidelinesProfileName);
|
||||
if (profile == null && fairProfile == null) {
|
||||
log.error("Exception: No valid guidelines");
|
||||
new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided.");
|
||||
}
|
||||
|
||||
AbstractOpenAireProfile profile = null;
|
||||
AbstractOpenAireProfile fairProfile = null;
|
||||
|
||||
profile = initializeProfile(guidelinesProfileName, profile, fairProfile);
|
||||
ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords);
|
||||
validationJob.guidelines = profile.name();
|
||||
System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n");
|
||||
validationJobRepository.save(validationJob);
|
||||
|
||||
//TODO uncomment
|
||||
// validationJobRepository.save(validationJob);
|
||||
int record = 0;
|
||||
double resultSum = 0;
|
||||
|
||||
|
|
@ -77,21 +78,17 @@ public class ValidationController {
|
|||
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||
|
||||
String baseURL = "http://repositorium.sdum.uminho.pt/oai/request"; // replace with your repository's base URL
|
||||
String metadataPrefix = "oai_dc"; // replace with your desired metadata prefix
|
||||
String set = null; // replace with your desired set, or null for all records
|
||||
int batchSize = 50; // replace with your desired batch size
|
||||
|
||||
// Step 1: Make initial request
|
||||
String url = baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix;
|
||||
if (set != null) {
|
||||
url += "&set=" + set;
|
||||
}
|
||||
// url += "&pageSize=" + batchSize;
|
||||
System.out.println("URL " + url.toString());
|
||||
|
||||
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
||||
Document oaipmhResponse = parseResponse(conn.getInputStream());
|
||||
List<String> records = new ArrayList<String>();
|
||||
|
||||
//TODO: follow other approach if records are more than 100!!!
|
||||
int recordsRetrieved = 0;
|
||||
// Step 2: Retrieve additional records using resumptionToken
|
||||
while (recordsRetrieved < batchSize) {
|
||||
|
|
@ -105,61 +102,49 @@ public class ValidationController {
|
|||
conn = (HttpURLConnection) new URL(url).openConnection();
|
||||
oaipmhResponse = parseResponse(conn.getInputStream());
|
||||
|
||||
// TODO: process the records returned in this response
|
||||
|
||||
|
||||
XPathFactory xfactory = XPathFactory.newInstance();
|
||||
XPath xpath = xfactory.newXPath();
|
||||
XPathExpression recordsExpression = xpath.compile("//record");
|
||||
NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET);
|
||||
|
||||
List<String> records = new ArrayList<String>();
|
||||
for (int i = 0; i < recordNodes.getLength(); ++i) {
|
||||
Node element = recordNodes.item(i);
|
||||
|
||||
StringWriter stringWriter = new StringWriter();
|
||||
Transformer xform = TransformerFactory.newInstance().newTransformer();
|
||||
xform.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); // optional
|
||||
xform.setOutputProperty(OutputKeys.INDENT, "yes"); // optional
|
||||
xform.transform(new DOMSource(element), new StreamResult(stringWriter));
|
||||
|
||||
records.add(stringWriter.toString());
|
||||
}
|
||||
|
||||
System.out.println("size " + records.size());
|
||||
recordsRetrieved += records.size();
|
||||
}
|
||||
|
||||
for (String recordXml : records) {
|
||||
Document doc = db.parse(new InputSource(new StringReader(recordXml)));
|
||||
if (profile != null) {
|
||||
for (String recordXml : records) {
|
||||
System.out.println("\n\n\n\n\n\n\n\n HELLLOOOOOOOOOO \n\n\n\n\n");
|
||||
Document doc = db.parse(new InputSource(new StringReader(recordXml)));
|
||||
System.out.println("\n\n\n\n RECORD : " + recordXml + "\n\n\n\n\n");
|
||||
|
||||
//what id is that?
|
||||
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc);
|
||||
if (profile != null) {
|
||||
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that?
|
||||
|
||||
Map<String, Guideline.Result> results = validationResult.results();
|
||||
/* for (Map.Entry entry : results.entrySet()) {
|
||||
|
||||
ValidationJobResult validationJobResult = new ValidationJobResult();
|
||||
validationJobResult.validationJobId = validationJob.id;
|
||||
validationJobResult.ruleName = entry.getKey().toString();
|
||||
validationJobResult.recordUrl = "localhost://records/record["+record+"]"; // silly id
|
||||
Map<String, Guideline.Result> results = validationResult.results();
|
||||
for (Map.Entry entry : results.entrySet()) {
|
||||
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||
validationJobResult.score = engineResult.score();
|
||||
validationJobResult.status = engineResult.status().toString();
|
||||
validationJobResult.internalError = engineResult.internalError();
|
||||
String recordUrl = extractRecordUrl(doc, "identifier");
|
||||
System.out.println("DC IDENTIFIER: " + recordUrl);
|
||||
String ruleName = entry.getKey().toString();
|
||||
ValidationRuleResult validationRuleResult = constructValidationRuleResult(validationJob.id, recordUrl,
|
||||
ruleName, profile, engineResult);
|
||||
|
||||
System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n");
|
||||
|
||||
resultSum += engineResult.score();
|
||||
|
||||
validationJobResult.warnings = ((List<String>) engineResult.warnings()).toArray(new String[0]);
|
||||
validationJobResult.errors = ((List<String>) engineResult.errors()).toArray(new String[0]);
|
||||
System.out.println(validationJobResult + " | " + validationJobResult.hashCode() + "\n");
|
||||
validationResultRepository.save(validationJobResult);
|
||||
validationJobResults.add(validationJobResult);
|
||||
|
||||
|
||||
}*/
|
||||
|
||||
validationResultRepository.save(validationRuleResult);
|
||||
saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult);
|
||||
}
|
||||
|
||||
record++;
|
||||
System.out.println(record);
|
||||
}
|
||||
}
|
||||
validationJob.status = "COMPLETED";
|
||||
|
|
@ -183,26 +168,115 @@ public class ValidationController {
|
|||
//xmlValidationResponse.setFairRules(fairRules);
|
||||
}
|
||||
|
||||
private AbstractOpenAireProfile initializeProfile(@RequestParam(name = "guidelines") String guidelinesProfileName, AbstractOpenAireProfile profile, AbstractOpenAireProfile fairProfile) {
|
||||
|
||||
private void constructValidationJobResult(ValidationJob validationJob, ValidationRuleResult validationRuleResult,
|
||||
Map.Entry entry, AbstractOpenAireProfile profile, Document document) {
|
||||
validationRuleResult.validationJobId = validationJob.id;
|
||||
validationRuleResult.ruleName = entry.getKey().toString();
|
||||
validationRuleResult.ruleWeight = profile.guideline(validationRuleResult.ruleName).getWeight();
|
||||
validationRuleResult.recordUrl = extractRecordUrl(document, "dc:identifier");
|
||||
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||
validationRuleResult.score = engineResult.score();
|
||||
validationRuleResult.status = engineResult.status().toString();
|
||||
validationRuleResult.internalError = engineResult.internalError();
|
||||
}
|
||||
|
||||
|
||||
|
||||
private ValidationRuleResult constructValidationJobResult(ValidationJob validationJob,
|
||||
Map.Entry entry, AbstractOpenAireProfile profile, Document document) {
|
||||
|
||||
ValidationRuleResult validationRuleResult = new ValidationRuleResult();
|
||||
validationRuleResult.validationJobId = validationJob.id;
|
||||
validationRuleResult.ruleName = entry.getKey().toString();
|
||||
validationRuleResult.ruleWeight = profile.guideline(validationRuleResult.ruleName).getWeight();
|
||||
validationRuleResult.recordUrl = extractRecordUrl(document, "dc:identifier");
|
||||
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||
validationRuleResult.score = engineResult.score();
|
||||
validationRuleResult.status = engineResult.status().toString();
|
||||
validationRuleResult.internalError = engineResult.internalError();
|
||||
return validationRuleResult;
|
||||
}
|
||||
|
||||
private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) {
|
||||
System.out.println("-------");
|
||||
for (String error:engineResult.errors()) {
|
||||
System.out.println("11111");
|
||||
ValidationIssue validationIssue = new ValidationIssue();
|
||||
validationIssue.validationJobId = validationJobId;
|
||||
validationIssue.ruleName = ruleName;
|
||||
validationIssue.recordUrl = recordUrl;
|
||||
validationIssue.issueType = "ERROR";
|
||||
validationIssue.issueText = error;
|
||||
validationIssueRepository.save(validationIssue);
|
||||
System.out.println(validationIssue);
|
||||
}
|
||||
|
||||
for (String warning: engineResult.warnings()){
|
||||
System.out.println("22222");
|
||||
ValidationIssue validationIssue = new ValidationIssue();
|
||||
validationIssue.validationJobId = validationJobId;
|
||||
validationIssue.ruleName = ruleName;
|
||||
validationIssue.recordUrl = recordUrl;
|
||||
validationIssue.issueType = "WARNING";
|
||||
validationIssue.issueText = warning;
|
||||
validationIssueRepository.save(validationIssue);
|
||||
System.out.println(validationIssue);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private ValidationRuleResult constructValidationRuleResult(int validationJobId, String recordUrl, String ruleName,
|
||||
AbstractOpenAireProfile profile, Guideline.Result engineResult) {
|
||||
|
||||
ValidationRuleResult validationRuleResult = new ValidationRuleResult();
|
||||
validationRuleResult.validationJobId = validationJobId;
|
||||
validationRuleResult.ruleName = ruleName;
|
||||
validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight();
|
||||
validationRuleResult.recordUrl = recordUrl;
|
||||
validationRuleResult.score = engineResult.score();
|
||||
validationRuleResult.status = engineResult.status().toString();
|
||||
validationRuleResult.internalError = engineResult.internalError();
|
||||
|
||||
return validationRuleResult;
|
||||
}
|
||||
|
||||
//TODO consider throwing exception
|
||||
private String extractRecordUrl(Document doc, String xmlField) {
|
||||
NodeList identifierNodes = doc.getElementsByTagName(xmlField);
|
||||
if (identifierNodes.getLength() > 0) {
|
||||
Element identifierElement = (Element) identifierNodes.item(0);
|
||||
return identifierElement.getTextContent();
|
||||
}
|
||||
return "-";
|
||||
}
|
||||
|
||||
private AbstractOpenAireProfile initializeOpenAireProfile(String guidelinesProfileName) {
|
||||
|
||||
if (guidelinesProfileName.equals("dataArchiveGuidelinesV2Profile")) {
|
||||
profile = new DataArchiveGuidelinesV2Profile();
|
||||
fairProfile = new FAIR_Data_GuidelinesProfile();
|
||||
return new DataArchiveGuidelinesV2Profile();
|
||||
|
||||
} else if (guidelinesProfileName.equals("literatureGuidelinesV3Profile")) {
|
||||
profile = new LiteratureGuidelinesV3Profile();
|
||||
return new LiteratureGuidelinesV3Profile();
|
||||
|
||||
} else if (guidelinesProfileName.equals("literatureGuidelinesV4Profile")) {
|
||||
profile = new LiteratureGuidelinesV4Profile();
|
||||
return new LiteratureGuidelinesV4Profile();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
private AbstractOpenAireProfile initializeFairProfile(String guidelinesProfileName) {
|
||||
if (guidelinesProfileName.equals("dataArchiveGuidelinesV2Profile")) {
|
||||
return new FAIR_Data_GuidelinesProfile();
|
||||
|
||||
} else if (guidelinesProfileName.equals("fairDataGuidelinesProfile")) {
|
||||
fairProfile = new FAIR_Data_GuidelinesProfile();
|
||||
return new FAIR_Data_GuidelinesProfile();
|
||||
}
|
||||
|
||||
if (profile == null && fairProfile == null) {
|
||||
log.error("Exception: No valid guidelines");
|
||||
new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided.");
|
||||
}
|
||||
return profile;
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -236,10 +310,8 @@ public class ValidationController {
|
|||
|
||||
List<RuleInfo> resultRules = null;
|
||||
List<RuleInfo> fairRules = null;
|
||||
AbstractOpenAireProfile profile = null;
|
||||
AbstractOpenAireProfile fairProfile = null;
|
||||
|
||||
profile = initializeProfile(guidelinesProfileName, profile, fairProfile);
|
||||
AbstractOpenAireProfile profile = initializeOpenAireProfile(guidelinesProfileName);
|
||||
AbstractOpenAireProfile fairProfile = initializeFairProfile(guidelinesProfileName);
|
||||
|
||||
validationJob.guidelines = profile.name();
|
||||
System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n");
|
||||
|
|
@ -249,7 +321,7 @@ public class ValidationController {
|
|||
|
||||
try {
|
||||
List<String> recordXmls = extractRecordXmls(OAIPMHResponse);
|
||||
List<ValidationJobResult> validationJobResults = new ArrayList<>();
|
||||
List<ValidationRuleResult> validationRuleResults = new ArrayList<>();
|
||||
|
||||
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||
|
|
@ -264,21 +336,19 @@ public class ValidationController {
|
|||
Map<String, Guideline.Result> results = validationResult.results();
|
||||
for (Map.Entry entry : results.entrySet()) {
|
||||
|
||||
ValidationJobResult validationJobResult = new ValidationJobResult();
|
||||
validationJobResult.validationJobId = validationJob.id;
|
||||
validationJobResult.ruleName = entry.getKey().toString();
|
||||
validationJobResult.recordUrl = "localhost://records/record["+record+"]"; // silly id
|
||||
ValidationRuleResult validationRuleResult = new ValidationRuleResult();
|
||||
validationRuleResult.validationJobId = validationJob.id;
|
||||
validationRuleResult.ruleName = entry.getKey().toString();
|
||||
validationRuleResult.recordUrl = "localhost://records/record["+record+"]"; // silly id
|
||||
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||
validationJobResult.score = engineResult.score();
|
||||
validationJobResult.status = engineResult.status().toString();
|
||||
validationJobResult.internalError = engineResult.internalError();
|
||||
resultSum += engineResult.score();
|
||||
validationRuleResult.score = engineResult.score();
|
||||
validationRuleResult.status = engineResult.status().toString();
|
||||
validationRuleResult.internalError = engineResult.internalError();
|
||||
System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n");
|
||||
validationResultRepository.save(validationRuleResult);
|
||||
|
||||
validationJobResult.warnings = ((List<String>) engineResult.warnings()).toArray(new String[0]);
|
||||
validationJobResult.errors = ((List<String>) engineResult.errors()).toArray(new String[0]);
|
||||
System.out.println(validationJobResult + " | " + validationJobResult.hashCode() + "\n");
|
||||
validationResultRepository.save(validationJobResult);
|
||||
validationJobResults.add(validationJobResult);
|
||||
validationRuleResults.add(validationRuleResult);
|
||||
resultSum += engineResult.score();
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -306,12 +376,89 @@ public class ValidationController {
|
|||
//xmlValidationResponse.setFairRules(fairRules);
|
||||
}
|
||||
|
||||
@RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.POST,
|
||||
@RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.GET,
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public List<SummaryResult> getJobResults(@RequestParam(name = "jobId") String jobId){
|
||||
System.out.println(validationResultRepository.getSummaryResult().toString());
|
||||
System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult().get(0));
|
||||
return validationResultRepository.getSummaryResult();
|
||||
public List<SummaryResult> getJobResults(@RequestParam(name = "jobId") int jobId){
|
||||
System.out.println(validationResultRepository.getSummaryResult(jobId).toString());
|
||||
System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult(jobId).get(0));
|
||||
return validationResultRepository.getSummaryResult(jobId);
|
||||
}
|
||||
/*
|
||||
@RequestMapping(value = {"getErrors"}, method = RequestMethod.GET,
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
|
||||
|
||||
List<Object[]> resultList = validationIssueRepository.getErrorsRecordUrls(jobId, ruleName);
|
||||
|
||||
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
|
||||
|
||||
for (Object[] result : resultList) {
|
||||
String issueText = (String) result[0];
|
||||
String recordUrl = (String) result[1];
|
||||
|
||||
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
|
||||
if (recordUrls == null) {
|
||||
recordUrls = new ArrayList<>();
|
||||
recordUrlsByIssueText.put(issueText, recordUrls);
|
||||
}
|
||||
recordUrls.add(recordUrl);
|
||||
}
|
||||
|
||||
System.out.println("issues " + recordUrlsByIssueText.keySet());
|
||||
|
||||
return recordUrlsByIssueText;
|
||||
}*/
|
||||
|
||||
@RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET,
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public List<ValidationIssue> getRecordsByRule(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
|
||||
return validationIssueRepository.getErrors(jobId, ruleName);
|
||||
}
|
||||
|
||||
@RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET,
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public Map<String, List<String>> getErrors(@RequestParam(name = "jobId") int jobId){
|
||||
List<Object[]> resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId);
|
||||
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
|
||||
|
||||
for (Object[] result : resultList) {
|
||||
String issueText = (String) result[0];
|
||||
String recordUrl = (String) result[1];
|
||||
|
||||
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
|
||||
if (recordUrls == null) {
|
||||
recordUrls = new ArrayList<>();
|
||||
recordUrlsByIssueText.put(issueText, recordUrls);
|
||||
}
|
||||
recordUrls.add(recordUrl);
|
||||
}
|
||||
|
||||
System.out.println("issues " + recordUrlsByIssueText.keySet());
|
||||
|
||||
return recordUrlsByIssueText;
|
||||
}
|
||||
|
||||
@RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET,
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public Map<String, List<String>> getWarningsReport(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){
|
||||
List<Object[]> resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId);
|
||||
Map<String, List<String>> recordUrlsByIssueText = new HashMap<>();
|
||||
|
||||
for (Object[] result : resultList) {
|
||||
String issueText = (String) result[0];
|
||||
String recordUrl = (String) result[1];
|
||||
|
||||
List<String> recordUrls = recordUrlsByIssueText.get(issueText);
|
||||
if (recordUrls == null) {
|
||||
recordUrls = new ArrayList<>();
|
||||
recordUrlsByIssueText.put(issueText, recordUrls);
|
||||
}
|
||||
recordUrls.add(recordUrl);
|
||||
}
|
||||
|
||||
System.out.println("issues " + recordUrlsByIssueText.keySet());
|
||||
|
||||
return recordUrlsByIssueText;
|
||||
}
|
||||
|
||||
public List<String> extractRecordXmls(String xml) throws Exception {
|
||||
|
|
|
|||
|
|
@ -10,27 +10,21 @@ public class SummaryResult {
|
|||
|
||||
@Id
|
||||
String rule_name;
|
||||
String rule_weight;
|
||||
int rule_weight;
|
||||
long passed_records;
|
||||
long failed_records;
|
||||
long errors;
|
||||
long warnings;
|
||||
|
||||
public SummaryResult(String rule_name, long passed_records, long failed_records, long errors, long warnings) {
|
||||
public SummaryResult(String rule_name, long passed_records, long failed_records) {
|
||||
this.rule_name = rule_name;
|
||||
this.passed_records = passed_records;
|
||||
this.failed_records = failed_records;
|
||||
this.errors = errors;
|
||||
this.warnings = warnings;
|
||||
}
|
||||
|
||||
public SummaryResult(String rule_name, String rule_weight, long passed_records, long failed_records, long errors, long warnings) {
|
||||
public SummaryResult(String rule_name, int rule_weight, long passed_records, long failed_records) {
|
||||
this.rule_name = rule_name;
|
||||
this.rule_weight = rule_weight;
|
||||
this.passed_records = passed_records;
|
||||
this.failed_records = failed_records;
|
||||
this.errors = errors;
|
||||
this.warnings = warnings;
|
||||
}
|
||||
|
||||
public String getRule_name() {
|
||||
|
|
@ -41,11 +35,11 @@ public class SummaryResult {
|
|||
this.rule_name = rule_name;
|
||||
}
|
||||
|
||||
public String getRule_weight() {
|
||||
public int getRule_weight() {
|
||||
return rule_weight;
|
||||
}
|
||||
|
||||
public void setRule_weight(String rule_weight) {
|
||||
public void setRule_weight(int rule_weight) {
|
||||
this.rule_weight = rule_weight;
|
||||
}
|
||||
|
||||
|
|
@ -65,22 +59,6 @@ public class SummaryResult {
|
|||
this.failed_records = failed_records;
|
||||
}
|
||||
|
||||
public long getErrors() {
|
||||
return errors;
|
||||
}
|
||||
|
||||
public void setErrors(long errors) {
|
||||
this.errors = errors;
|
||||
}
|
||||
|
||||
public long getWarnings() {
|
||||
return warnings;
|
||||
}
|
||||
|
||||
public void setWarnings(long warnings) {
|
||||
this.warnings = warnings;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "SummaryResult{" +
|
||||
|
|
@ -88,8 +66,6 @@ public class SummaryResult {
|
|||
", rule_weight=" + rule_weight +
|
||||
", passed_records=" + passed_records +
|
||||
", failed_records=" + failed_records +
|
||||
", errors=" + errors +
|
||||
", warnings=" + warnings +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
package eu.dnetlib.validatorapi.entities;
|
||||
|
||||
import javax.persistence.*;
|
||||
import java.io.Serializable;
|
||||
|
||||
@Entity
|
||||
@Table(name="validation_issues")
|
||||
@IdClass(ValidationIssue.class)
|
||||
public class ValidationIssue implements Serializable {
|
||||
@Id
|
||||
@Column(name = "validation_job_id")
|
||||
public int validationJobId;
|
||||
|
||||
@Id
|
||||
@Column(name = "rule_name")
|
||||
public String ruleName;
|
||||
|
||||
@Id
|
||||
@Column(name = "record_url")
|
||||
public String recordUrl;
|
||||
|
||||
@Id
|
||||
@Column(name = "issue_text")
|
||||
public String issueText;
|
||||
|
||||
@Column(name = "issue_type")
|
||||
public String issueType;
|
||||
|
||||
public int getValidationJobId() {
|
||||
return validationJobId;
|
||||
}
|
||||
|
||||
public void setValidationJobId(int validationJobId) {
|
||||
this.validationJobId = validationJobId;
|
||||
}
|
||||
|
||||
public String getRuleName() {
|
||||
return ruleName;
|
||||
}
|
||||
|
||||
public void setRuleName(String ruleName) {
|
||||
this.ruleName = ruleName;
|
||||
}
|
||||
|
||||
public String getRecordUrl() {
|
||||
return recordUrl;
|
||||
}
|
||||
|
||||
public void setRecordUrl(String recordUrl) {
|
||||
this.recordUrl = recordUrl;
|
||||
}
|
||||
|
||||
public String getIssueText() {
|
||||
return issueText;
|
||||
}
|
||||
|
||||
public void setIssueText(String issueText) {
|
||||
this.issueText = issueText;
|
||||
}
|
||||
|
||||
public String getIssueType() {
|
||||
return issueType;
|
||||
}
|
||||
|
||||
public void setIssueType(String issueType) {
|
||||
this.issueType = issueType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ValidationIssue{" +
|
||||
"validationJobId=" + validationJobId +
|
||||
", ruleName='" + ruleName + '\'' +
|
||||
", recordUrl='" + recordUrl + '\'' +
|
||||
", issueText='" + issueText + '\'' +
|
||||
", issueType='" + issueType + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,5 @@
|
|||
package eu.dnetlib.validatorapi.entities;
|
||||
|
||||
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
||||
|
||||
import javax.persistence.*;
|
||||
import java.util.Date;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,12 @@
|
|||
package eu.dnetlib.validatorapi.entities;
|
||||
|
||||
import eu.dnetlib.validator2.engine.Status;
|
||||
import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
||||
import org.hibernate.annotations.Type;
|
||||
|
||||
import javax.persistence.*;
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
|
||||
@Entity
|
||||
@Table(name="validation_results")
|
||||
@IdClass(ValidationJobResult.class)
|
||||
public class ValidationJobResult implements Serializable {
|
||||
@IdClass(ValidationRuleResult.class)
|
||||
public class ValidationRuleResult implements Serializable {
|
||||
|
||||
@Id
|
||||
@Column(name = "validation_job_id")
|
||||
|
|
@ -23,36 +18,31 @@ public class ValidationJobResult implements Serializable {
|
|||
|
||||
@Id
|
||||
@Column(name = "rule_weight")
|
||||
public String ruleWeight;
|
||||
public int ruleWeight;
|
||||
|
||||
@Id
|
||||
@Column(name = "record_url")
|
||||
public String recordUrl;
|
||||
|
||||
//not in use yet
|
||||
@Column(name = "warnings", columnDefinition = "text[]")
|
||||
@Type(type = "eu.dnetlib.validatorapi.utils.CustomStringArrayType")
|
||||
public String[] warnings;
|
||||
|
||||
@Column(name = "errors", columnDefinition = "text[]")
|
||||
@Type(type = "eu.dnetlib.validatorapi.utils.CustomStringArrayType")
|
||||
public String[] errors;
|
||||
|
||||
@Column(name = "internal_error")
|
||||
public String internalError;
|
||||
|
||||
@Column(name = "status")
|
||||
public String status;
|
||||
|
||||
@Column(name = "score")
|
||||
public double score;
|
||||
|
||||
public ValidationJobResult() {}
|
||||
public ValidationRuleResult() {}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ValidationJobResult{" +
|
||||
", warnings=" + warnings +
|
||||
", errors=" + errors +
|
||||
"validationJobId=" + validationJobId +
|
||||
", ruleName='" + ruleName + '\'' +
|
||||
", ruleWeight='" + ruleWeight + '\'' +
|
||||
", recordUrl='" + recordUrl + '\'' +
|
||||
", internalError='" + internalError + '\'' +
|
||||
", status='" + status + '\'' +
|
||||
", score=" + score +
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package eu.dnetlib.validatorapi.repositories;
|
||||
|
||||
import eu.dnetlib.validatorapi.entities.SummaryResult;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationIssue;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import javax.persistence.EntityManager;
|
||||
import javax.persistence.PersistenceContext;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Repository
|
||||
public interface ValidationIssueRepository extends JpaRepository<ValidationIssue, Long> {
|
||||
|
||||
|
||||
@Query(value = "SELECT vi.issueText, vi.recordUrl " +
|
||||
"FROM ValidationIssue vi " +
|
||||
"WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id AND vi.ruleName =:ruleName)" +
|
||||
"GROUP BY vi.issueText, vi.recordUrl")
|
||||
List<ValidationIssue> getWarnings(@Param("id") int validationJobId, @Param("ruleName") String ruleName);
|
||||
|
||||
@Query(value = "SELECT vi.issueText, vi.recordUrl " +
|
||||
"FROM ValidationIssue vi " +
|
||||
"WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id AND vi.ruleName =:ruleName)" +
|
||||
"GROUP BY vi.issueText, vi.recordUrl")
|
||||
List<ValidationIssue> getErrors(@Param("id") int validationJobId, @Param("ruleName") String ruleName);
|
||||
|
||||
|
||||
@Query(value = "SELECT vi.issueText, vi.recordUrl " +
|
||||
"FROM ValidationIssue vi " +
|
||||
"WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" +
|
||||
"GROUP BY vi.issueText, vi.recordUrl")
|
||||
List<Object[]> getAllErrorsRecordUrls(@Param("id") int validationJobId);
|
||||
|
||||
@Query(value = "SELECT vi.issueText, vi.recordUrl " +
|
||||
"FROM ValidationIssue vi " +
|
||||
"WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id)" +
|
||||
"GROUP BY vi.issueText, vi.recordUrl")
|
||||
List<Object[]> getAllWarningsRecordUrls(@Param("id") int validationJobId);
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.validatorapi.repository;
|
||||
package eu.dnetlib.validatorapi.repositories;
|
||||
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
|
@ -1,23 +1,22 @@
|
|||
package eu.dnetlib.validatorapi.repository;
|
||||
package eu.dnetlib.validatorapi.repositories;
|
||||
|
||||
import eu.dnetlib.validatorapi.entities.SummaryResult;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJobResult;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationRuleResult;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Repository
|
||||
public interface ValidationResultRepository extends JpaRepository<ValidationJobResult, Long> {
|
||||
public interface ValidationResultRepository extends JpaRepository<ValidationRuleResult, Long> {
|
||||
|
||||
@Query(value =
|
||||
"SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, sr.ruleWeight," +
|
||||
"COUNT(CASE WHEN sr.status = 'SUCCESS' THEN 1 END) AS passed_records, " +
|
||||
"COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failed_records, " +
|
||||
"COUNT(CASE WHEN array_length(sr.errors, 1) > 0 THEN 1 END) AS errors, " +
|
||||
"COUNT(CASE WHEN array_length(sr.warnings, 1) > 0 THEN 1 END) AS warnings) " +
|
||||
"FROM ValidationJobResult sr " +
|
||||
"COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failed_records) " +
|
||||
"FROM ValidationRuleResult sr " + "WHERE sr.validationJobId = :id " +
|
||||
"GROUP BY sr.ruleName, sr.ruleWeight")
|
||||
List<SummaryResult> getSummaryResult();
|
||||
List<SummaryResult> getSummaryResult(@Param("id") int validationJobId);
|
||||
}
|
||||
Loading…
Reference in New Issue