From a15670b741c3b3375b66a95dac8921a851b769aa Mon Sep 17 00:00:00 2001 From: Katerina Date: Wed, 3 May 2023 15:00:57 +0300 Subject: [PATCH] progress with error and warning reports. not yet final. need to clean code and finalize the needed methods --- .../controllers/ValidationController.java | 331 +++++++++++++----- .../validatorapi/entities/SummaryResult.java | 34 +- .../entities/ValidationIssue.java | 79 +++++ .../validatorapi/entities/ValidationJob.java | 2 - ...bResult.java => ValidationRuleResult.java} | 30 +- .../ValidationIssueRepository.java | 46 +++ .../ValidationJobRepository.java | 2 +- .../ValidationResultRepository.java | 15 +- 8 files changed, 387 insertions(+), 152 deletions(-) create mode 100644 src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java rename src/main/java/eu/dnetlib/validatorapi/entities/{ValidationJobResult.java => ValidationRuleResult.java} (52%) create mode 100644 src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java rename src/main/java/eu/dnetlib/validatorapi/{repository => repositories}/ValidationJobRepository.java (85%) rename src/main/java/eu/dnetlib/validatorapi/{repository => repositories}/ValidationResultRepository.java (60%) diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index 09fa5a4..456be40 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -3,12 +3,10 @@ package eu.dnetlib.validatorapi.controllers; import eu.dnetlib.validator2.validation.XMLApplicationProfile; import eu.dnetlib.validator2.validation.guideline.Guideline; import eu.dnetlib.validator2.validation.guideline.openaire.*; -import eu.dnetlib.validatorapi.entities.RuleInfo; -import eu.dnetlib.validatorapi.entities.SummaryResult; -import eu.dnetlib.validatorapi.entities.ValidationJob; -import eu.dnetlib.validatorapi.entities.ValidationJobResult; -import eu.dnetlib.validatorapi.repository.ValidationJobRepository; -import eu.dnetlib.validatorapi.repository.ValidationResultRepository; +import eu.dnetlib.validatorapi.entities.*; +import eu.dnetlib.validatorapi.repositories.ValidationIssueRepository; +import eu.dnetlib.validatorapi.repositories.ValidationJobRepository; +import eu.dnetlib.validatorapi.repositories.ValidationResultRepository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -45,30 +43,33 @@ public class ValidationController { private final Logger log = LogManager.getLogger(this.getClass()); private final ValidationJobRepository validationJobRepository; private final ValidationResultRepository validationResultRepository; + private final ValidationIssueRepository validationIssueRepository; @Autowired - public ValidationController(ValidationJobRepository validationJobRepository, ValidationResultRepository validationResultRepository) { + public ValidationController(ValidationJobRepository validationJobRepository, ValidationResultRepository validationResultRepository, ValidationIssueRepository validationIssueRepository) { this.validationJobRepository = validationJobRepository; this.validationResultRepository = validationResultRepository; + this.validationIssueRepository = validationIssueRepository; } @RequestMapping(value = {"/realValidator"}, method = RequestMethod.POST) public void validateRealOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName, - @RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now - @RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords //not in use now - ) { + @RequestParam(name = "baseUrl", defaultValue = "http://repositorium.sdum.uminho.pt/oai/request") String baseURL, //not in use now + @RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now + @RequestParam(name="set", required = false) String set, @RequestParam(name="metadataPrefix", defaultValue = "oai_dc") String metadataPrefix, + @RequestParam(name="batchsize", defaultValue = "50") int batchSize) { - ValidationJob validationJob = new ValidationJob(baseUrl, numberOfRecords); + AbstractOpenAireProfile profile = initializeOpenAireProfile(guidelinesProfileName); + AbstractOpenAireProfile fairProfile = initializeFairProfile(guidelinesProfileName); + if (profile == null && fairProfile == null) { + log.error("Exception: No valid guidelines"); + new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided."); + } - AbstractOpenAireProfile profile = null; - AbstractOpenAireProfile fairProfile = null; - - profile = initializeProfile(guidelinesProfileName, profile, fairProfile); + ValidationJob validationJob = new ValidationJob(baseURL, numberOfRecords); validationJob.guidelines = profile.name(); - System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n"); + validationJobRepository.save(validationJob); - //TODO uncomment - // validationJobRepository.save(validationJob); int record = 0; double resultSum = 0; @@ -77,21 +78,17 @@ public class ValidationController { DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); - String baseURL = "http://repositorium.sdum.uminho.pt/oai/request"; // replace with your repository's base URL - String metadataPrefix = "oai_dc"; // replace with your desired metadata prefix - String set = null; // replace with your desired set, or null for all records - int batchSize = 50; // replace with your desired batch size - - // Step 1: Make initial request String url = baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix; if (set != null) { url += "&set=" + set; } - // url += "&pageSize=" + batchSize; System.out.println("URL " + url.toString()); + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); Document oaipmhResponse = parseResponse(conn.getInputStream()); + List records = new ArrayList(); + //TODO: follow other approach if records are more than 100!!! int recordsRetrieved = 0; // Step 2: Retrieve additional records using resumptionToken while (recordsRetrieved < batchSize) { @@ -105,61 +102,49 @@ public class ValidationController { conn = (HttpURLConnection) new URL(url).openConnection(); oaipmhResponse = parseResponse(conn.getInputStream()); - // TODO: process the records returned in this response - - XPathFactory xfactory = XPathFactory.newInstance(); XPath xpath = xfactory.newXPath(); XPathExpression recordsExpression = xpath.compile("//record"); NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET); - List records = new ArrayList(); for (int i = 0; i < recordNodes.getLength(); ++i) { Node element = recordNodes.item(i); - StringWriter stringWriter = new StringWriter(); Transformer xform = TransformerFactory.newInstance().newTransformer(); - xform.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); // optional - xform.setOutputProperty(OutputKeys.INDENT, "yes"); // optional xform.transform(new DOMSource(element), new StreamResult(stringWriter)); - records.add(stringWriter.toString()); } - System.out.println("size " + records.size()); recordsRetrieved += records.size(); + } - for (String recordXml : records) { - Document doc = db.parse(new InputSource(new StringReader(recordXml))); - if (profile != null) { + for (String recordXml : records) { + System.out.println("\n\n\n\n\n\n\n\n HELLLOOOOOOOOOO \n\n\n\n\n"); + Document doc = db.parse(new InputSource(new StringReader(recordXml))); + System.out.println("\n\n\n\n RECORD : " + recordXml + "\n\n\n\n\n"); - //what id is that? - XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); + if (profile != null) { + XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); //what id is that? - Map results = validationResult.results(); - /* for (Map.Entry entry : results.entrySet()) { - - ValidationJobResult validationJobResult = new ValidationJobResult(); - validationJobResult.validationJobId = validationJob.id; - validationJobResult.ruleName = entry.getKey().toString(); - validationJobResult.recordUrl = "localhost://records/record["+record+"]"; // silly id + Map results = validationResult.results(); + for (Map.Entry entry : results.entrySet()) { Guideline.Result engineResult = (Guideline.Result) entry.getValue(); - validationJobResult.score = engineResult.score(); - validationJobResult.status = engineResult.status().toString(); - validationJobResult.internalError = engineResult.internalError(); + String recordUrl = extractRecordUrl(doc, "identifier"); + System.out.println("DC IDENTIFIER: " + recordUrl); + String ruleName = entry.getKey().toString(); + ValidationRuleResult validationRuleResult = constructValidationRuleResult(validationJob.id, recordUrl, + ruleName, profile, engineResult); + + System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n"); + resultSum += engineResult.score(); - validationJobResult.warnings = ((List) engineResult.warnings()).toArray(new String[0]); - validationJobResult.errors = ((List) engineResult.errors()).toArray(new String[0]); - System.out.println(validationJobResult + " | " + validationJobResult.hashCode() + "\n"); - validationResultRepository.save(validationJobResult); - validationJobResults.add(validationJobResult); - - - }*/ - + validationResultRepository.save(validationRuleResult); + saveValidationIssues(validationJob.id, recordUrl, ruleName, engineResult); } + record++; + System.out.println(record); } } validationJob.status = "COMPLETED"; @@ -183,26 +168,115 @@ public class ValidationController { //xmlValidationResponse.setFairRules(fairRules); } - private AbstractOpenAireProfile initializeProfile(@RequestParam(name = "guidelines") String guidelinesProfileName, AbstractOpenAireProfile profile, AbstractOpenAireProfile fairProfile) { + + private void constructValidationJobResult(ValidationJob validationJob, ValidationRuleResult validationRuleResult, + Map.Entry entry, AbstractOpenAireProfile profile, Document document) { + validationRuleResult.validationJobId = validationJob.id; + validationRuleResult.ruleName = entry.getKey().toString(); + validationRuleResult.ruleWeight = profile.guideline(validationRuleResult.ruleName).getWeight(); + validationRuleResult.recordUrl = extractRecordUrl(document, "dc:identifier"); + Guideline.Result engineResult = (Guideline.Result) entry.getValue(); + validationRuleResult.score = engineResult.score(); + validationRuleResult.status = engineResult.status().toString(); + validationRuleResult.internalError = engineResult.internalError(); + } + + + + private ValidationRuleResult constructValidationJobResult(ValidationJob validationJob, + Map.Entry entry, AbstractOpenAireProfile profile, Document document) { + + ValidationRuleResult validationRuleResult = new ValidationRuleResult(); + validationRuleResult.validationJobId = validationJob.id; + validationRuleResult.ruleName = entry.getKey().toString(); + validationRuleResult.ruleWeight = profile.guideline(validationRuleResult.ruleName).getWeight(); + validationRuleResult.recordUrl = extractRecordUrl(document, "dc:identifier"); + Guideline.Result engineResult = (Guideline.Result) entry.getValue(); + validationRuleResult.score = engineResult.score(); + validationRuleResult.status = engineResult.status().toString(); + validationRuleResult.internalError = engineResult.internalError(); + return validationRuleResult; + } + + private void saveValidationIssues(int validationJobId, String recordUrl, String ruleName, Guideline.Result engineResult) { + System.out.println("-------"); + for (String error:engineResult.errors()) { + System.out.println("11111"); + ValidationIssue validationIssue = new ValidationIssue(); + validationIssue.validationJobId = validationJobId; + validationIssue.ruleName = ruleName; + validationIssue.recordUrl = recordUrl; + validationIssue.issueType = "ERROR"; + validationIssue.issueText = error; + validationIssueRepository.save(validationIssue); + System.out.println(validationIssue); + } + + for (String warning: engineResult.warnings()){ + System.out.println("22222"); + ValidationIssue validationIssue = new ValidationIssue(); + validationIssue.validationJobId = validationJobId; + validationIssue.ruleName = ruleName; + validationIssue.recordUrl = recordUrl; + validationIssue.issueType = "WARNING"; + validationIssue.issueText = warning; + validationIssueRepository.save(validationIssue); + System.out.println(validationIssue); + + } + + } + + private ValidationRuleResult constructValidationRuleResult(int validationJobId, String recordUrl, String ruleName, + AbstractOpenAireProfile profile, Guideline.Result engineResult) { + + ValidationRuleResult validationRuleResult = new ValidationRuleResult(); + validationRuleResult.validationJobId = validationJobId; + validationRuleResult.ruleName = ruleName; + validationRuleResult.ruleWeight = profile.guideline(ruleName).getWeight(); + validationRuleResult.recordUrl = recordUrl; + validationRuleResult.score = engineResult.score(); + validationRuleResult.status = engineResult.status().toString(); + validationRuleResult.internalError = engineResult.internalError(); + + return validationRuleResult; + } + + //TODO consider throwing exception + private String extractRecordUrl(Document doc, String xmlField) { + NodeList identifierNodes = doc.getElementsByTagName(xmlField); + if (identifierNodes.getLength() > 0) { + Element identifierElement = (Element) identifierNodes.item(0); + return identifierElement.getTextContent(); + } + return "-"; + } + + private AbstractOpenAireProfile initializeOpenAireProfile(String guidelinesProfileName) { + if (guidelinesProfileName.equals("dataArchiveGuidelinesV2Profile")) { - profile = new DataArchiveGuidelinesV2Profile(); - fairProfile = new FAIR_Data_GuidelinesProfile(); + return new DataArchiveGuidelinesV2Profile(); } else if (guidelinesProfileName.equals("literatureGuidelinesV3Profile")) { - profile = new LiteratureGuidelinesV3Profile(); + return new LiteratureGuidelinesV3Profile(); } else if (guidelinesProfileName.equals("literatureGuidelinesV4Profile")) { - profile = new LiteratureGuidelinesV4Profile(); + return new LiteratureGuidelinesV4Profile(); + } + + return null; + } + + + private AbstractOpenAireProfile initializeFairProfile(String guidelinesProfileName) { + if (guidelinesProfileName.equals("dataArchiveGuidelinesV2Profile")) { + return new FAIR_Data_GuidelinesProfile(); } else if (guidelinesProfileName.equals("fairDataGuidelinesProfile")) { - fairProfile = new FAIR_Data_GuidelinesProfile(); + return new FAIR_Data_GuidelinesProfile(); } - if (profile == null && fairProfile == null) { - log.error("Exception: No valid guidelines"); - new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided."); - } - return profile; + return null; } @@ -236,10 +310,8 @@ public class ValidationController { List resultRules = null; List fairRules = null; - AbstractOpenAireProfile profile = null; - AbstractOpenAireProfile fairProfile = null; - - profile = initializeProfile(guidelinesProfileName, profile, fairProfile); + AbstractOpenAireProfile profile = initializeOpenAireProfile(guidelinesProfileName); + AbstractOpenAireProfile fairProfile = initializeFairProfile(guidelinesProfileName); validationJob.guidelines = profile.name(); System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n"); @@ -249,7 +321,7 @@ public class ValidationController { try { List recordXmls = extractRecordXmls(OAIPMHResponse); - List validationJobResults = new ArrayList<>(); + List validationRuleResults = new ArrayList<>(); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); @@ -264,21 +336,19 @@ public class ValidationController { Map results = validationResult.results(); for (Map.Entry entry : results.entrySet()) { - ValidationJobResult validationJobResult = new ValidationJobResult(); - validationJobResult.validationJobId = validationJob.id; - validationJobResult.ruleName = entry.getKey().toString(); - validationJobResult.recordUrl = "localhost://records/record["+record+"]"; // silly id + ValidationRuleResult validationRuleResult = new ValidationRuleResult(); + validationRuleResult.validationJobId = validationJob.id; + validationRuleResult.ruleName = entry.getKey().toString(); + validationRuleResult.recordUrl = "localhost://records/record["+record+"]"; // silly id Guideline.Result engineResult = (Guideline.Result) entry.getValue(); - validationJobResult.score = engineResult.score(); - validationJobResult.status = engineResult.status().toString(); - validationJobResult.internalError = engineResult.internalError(); - resultSum += engineResult.score(); + validationRuleResult.score = engineResult.score(); + validationRuleResult.status = engineResult.status().toString(); + validationRuleResult.internalError = engineResult.internalError(); + System.out.println(validationRuleResult + " | " + validationRuleResult.hashCode() + "\n"); + validationResultRepository.save(validationRuleResult); - validationJobResult.warnings = ((List) engineResult.warnings()).toArray(new String[0]); - validationJobResult.errors = ((List) engineResult.errors()).toArray(new String[0]); - System.out.println(validationJobResult + " | " + validationJobResult.hashCode() + "\n"); - validationResultRepository.save(validationJobResult); - validationJobResults.add(validationJobResult); + validationRuleResults.add(validationRuleResult); + resultSum += engineResult.score(); } @@ -306,12 +376,89 @@ public class ValidationController { //xmlValidationResponse.setFairRules(fairRules); } - @RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.POST, + @RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) - public List getJobResults(@RequestParam(name = "jobId") String jobId){ - System.out.println(validationResultRepository.getSummaryResult().toString()); - System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult().get(0)); - return validationResultRepository.getSummaryResult(); + public List getJobResults(@RequestParam(name = "jobId") int jobId){ + System.out.println(validationResultRepository.getSummaryResult(jobId).toString()); + System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult(jobId).get(0)); + return validationResultRepository.getSummaryResult(jobId); + } +/* + @RequestMapping(value = {"getErrors"}, method = RequestMethod.GET, + produces = MediaType.APPLICATION_JSON_VALUE) + public Map> getErrors(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ + + List resultList = validationIssueRepository.getErrorsRecordUrls(jobId, ruleName); + + Map> recordUrlsByIssueText = new HashMap<>(); + + for (Object[] result : resultList) { + String issueText = (String) result[0]; + String recordUrl = (String) result[1]; + + List recordUrls = recordUrlsByIssueText.get(issueText); + if (recordUrls == null) { + recordUrls = new ArrayList<>(); + recordUrlsByIssueText.put(issueText, recordUrls); + } + recordUrls.add(recordUrl); + } + + System.out.println("issues " + recordUrlsByIssueText.keySet()); + + return recordUrlsByIssueText; + }*/ + + @RequestMapping(value = {"getRecordsByRule"}, method = RequestMethod.GET, + produces = MediaType.APPLICATION_JSON_VALUE) + public List getRecordsByRule(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ + return validationIssueRepository.getErrors(jobId, ruleName); + } + + @RequestMapping(value = {"getErrorsReport"}, method = RequestMethod.GET, + produces = MediaType.APPLICATION_JSON_VALUE) + public Map> getErrors(@RequestParam(name = "jobId") int jobId){ + List resultList = validationIssueRepository.getAllErrorsRecordUrls(jobId); + Map> recordUrlsByIssueText = new HashMap<>(); + + for (Object[] result : resultList) { + String issueText = (String) result[0]; + String recordUrl = (String) result[1]; + + List recordUrls = recordUrlsByIssueText.get(issueText); + if (recordUrls == null) { + recordUrls = new ArrayList<>(); + recordUrlsByIssueText.put(issueText, recordUrls); + } + recordUrls.add(recordUrl); + } + + System.out.println("issues " + recordUrlsByIssueText.keySet()); + + return recordUrlsByIssueText; + } + + @RequestMapping(value = {"getWarningsReport"}, method = RequestMethod.GET, + produces = MediaType.APPLICATION_JSON_VALUE) + public Map> getWarningsReport(@RequestParam(name = "jobId") int jobId, @RequestParam(name = "ruleName") String ruleName){ + List resultList = validationIssueRepository.getAllWarningsRecordUrls(jobId); + Map> recordUrlsByIssueText = new HashMap<>(); + + for (Object[] result : resultList) { + String issueText = (String) result[0]; + String recordUrl = (String) result[1]; + + List recordUrls = recordUrlsByIssueText.get(issueText); + if (recordUrls == null) { + recordUrls = new ArrayList<>(); + recordUrlsByIssueText.put(issueText, recordUrls); + } + recordUrls.add(recordUrl); + } + + System.out.println("issues " + recordUrlsByIssueText.keySet()); + + return recordUrlsByIssueText; } public List extractRecordXmls(String xml) throws Exception { diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java b/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java index 72060e6..9c4b781 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java @@ -10,27 +10,21 @@ public class SummaryResult { @Id String rule_name; - String rule_weight; + int rule_weight; long passed_records; long failed_records; - long errors; - long warnings; - public SummaryResult(String rule_name, long passed_records, long failed_records, long errors, long warnings) { + public SummaryResult(String rule_name, long passed_records, long failed_records) { this.rule_name = rule_name; this.passed_records = passed_records; this.failed_records = failed_records; - this.errors = errors; - this.warnings = warnings; } - public SummaryResult(String rule_name, String rule_weight, long passed_records, long failed_records, long errors, long warnings) { + public SummaryResult(String rule_name, int rule_weight, long passed_records, long failed_records) { this.rule_name = rule_name; this.rule_weight = rule_weight; this.passed_records = passed_records; this.failed_records = failed_records; - this.errors = errors; - this.warnings = warnings; } public String getRule_name() { @@ -41,11 +35,11 @@ public class SummaryResult { this.rule_name = rule_name; } - public String getRule_weight() { + public int getRule_weight() { return rule_weight; } - public void setRule_weight(String rule_weight) { + public void setRule_weight(int rule_weight) { this.rule_weight = rule_weight; } @@ -65,22 +59,6 @@ public class SummaryResult { this.failed_records = failed_records; } - public long getErrors() { - return errors; - } - - public void setErrors(long errors) { - this.errors = errors; - } - - public long getWarnings() { - return warnings; - } - - public void setWarnings(long warnings) { - this.warnings = warnings; - } - @Override public String toString() { return "SummaryResult{" + @@ -88,8 +66,6 @@ public class SummaryResult { ", rule_weight=" + rule_weight + ", passed_records=" + passed_records + ", failed_records=" + failed_records + - ", errors=" + errors + - ", warnings=" + warnings + '}'; } diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java new file mode 100644 index 0000000..c9fba9a --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationIssue.java @@ -0,0 +1,79 @@ +package eu.dnetlib.validatorapi.entities; + +import javax.persistence.*; +import java.io.Serializable; + +@Entity +@Table(name="validation_issues") +@IdClass(ValidationIssue.class) +public class ValidationIssue implements Serializable { + @Id + @Column(name = "validation_job_id") + public int validationJobId; + + @Id + @Column(name = "rule_name") + public String ruleName; + + @Id + @Column(name = "record_url") + public String recordUrl; + + @Id + @Column(name = "issue_text") + public String issueText; + + @Column(name = "issue_type") + public String issueType; + + public int getValidationJobId() { + return validationJobId; + } + + public void setValidationJobId(int validationJobId) { + this.validationJobId = validationJobId; + } + + public String getRuleName() { + return ruleName; + } + + public void setRuleName(String ruleName) { + this.ruleName = ruleName; + } + + public String getRecordUrl() { + return recordUrl; + } + + public void setRecordUrl(String recordUrl) { + this.recordUrl = recordUrl; + } + + public String getIssueText() { + return issueText; + } + + public void setIssueText(String issueText) { + this.issueText = issueText; + } + + public String getIssueType() { + return issueType; + } + + public void setIssueType(String issueType) { + this.issueType = issueType; + } + + @Override + public String toString() { + return "ValidationIssue{" + + "validationJobId=" + validationJobId + + ", ruleName='" + ruleName + '\'' + + ", recordUrl='" + recordUrl + '\'' + + ", issueText='" + issueText + '\'' + + ", issueType='" + issueType + '\'' + + '}'; + } +} diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java index aab0c08..a128bd9 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJob.java @@ -1,7 +1,5 @@ package eu.dnetlib.validatorapi.entities; -import eu.dnetlib.validator2.validation.guideline.Guideline; - import javax.persistence.*; import java.util.Date; diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJobResult.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationRuleResult.java similarity index 52% rename from src/main/java/eu/dnetlib/validatorapi/entities/ValidationJobResult.java rename to src/main/java/eu/dnetlib/validatorapi/entities/ValidationRuleResult.java index 2187200..186da1b 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJobResult.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationRuleResult.java @@ -1,17 +1,12 @@ package eu.dnetlib.validatorapi.entities; -import eu.dnetlib.validator2.engine.Status; -import eu.dnetlib.validator2.validation.XMLApplicationProfile; -import org.hibernate.annotations.Type; - import javax.persistence.*; import java.io.Serializable; -import java.util.List; @Entity @Table(name="validation_results") -@IdClass(ValidationJobResult.class) -public class ValidationJobResult implements Serializable { +@IdClass(ValidationRuleResult.class) +public class ValidationRuleResult implements Serializable { @Id @Column(name = "validation_job_id") @@ -23,36 +18,31 @@ public class ValidationJobResult implements Serializable { @Id @Column(name = "rule_weight") - public String ruleWeight; + public int ruleWeight; @Id @Column(name = "record_url") public String recordUrl; - //not in use yet - @Column(name = "warnings", columnDefinition = "text[]") - @Type(type = "eu.dnetlib.validatorapi.utils.CustomStringArrayType") - public String[] warnings; - - @Column(name = "errors", columnDefinition = "text[]") - @Type(type = "eu.dnetlib.validatorapi.utils.CustomStringArrayType") - public String[] errors; - @Column(name = "internal_error") public String internalError; + @Column(name = "status") public String status; + @Column(name = "score") public double score; - public ValidationJobResult() {} + public ValidationRuleResult() {} @Override public String toString() { return "ValidationJobResult{" + - ", warnings=" + warnings + - ", errors=" + errors + + "validationJobId=" + validationJobId + + ", ruleName='" + ruleName + '\'' + + ", ruleWeight='" + ruleWeight + '\'' + + ", recordUrl='" + recordUrl + '\'' + ", internalError='" + internalError + '\'' + ", status='" + status + '\'' + ", score=" + score + diff --git a/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java new file mode 100644 index 0000000..626016b --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationIssueRepository.java @@ -0,0 +1,46 @@ +package eu.dnetlib.validatorapi.repositories; + +import eu.dnetlib.validatorapi.entities.SummaryResult; +import eu.dnetlib.validatorapi.entities.ValidationIssue; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Repository +public interface ValidationIssueRepository extends JpaRepository { + + + @Query(value = "SELECT vi.issueText, vi.recordUrl " + + "FROM ValidationIssue vi " + + "WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id AND vi.ruleName =:ruleName)" + + "GROUP BY vi.issueText, vi.recordUrl") + List getWarnings(@Param("id") int validationJobId, @Param("ruleName") String ruleName); + + @Query(value = "SELECT vi.issueText, vi.recordUrl " + + "FROM ValidationIssue vi " + + "WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id AND vi.ruleName =:ruleName)" + + "GROUP BY vi.issueText, vi.recordUrl") + List getErrors(@Param("id") int validationJobId, @Param("ruleName") String ruleName); + + + @Query(value = "SELECT vi.issueText, vi.recordUrl " + + "FROM ValidationIssue vi " + + "WHERE (vi.issueType = 'ERROR' AND vi.validationJobId =:id)" + + "GROUP BY vi.issueText, vi.recordUrl") + List getAllErrorsRecordUrls(@Param("id") int validationJobId); + + @Query(value = "SELECT vi.issueText, vi.recordUrl " + + "FROM ValidationIssue vi " + + "WHERE (vi.issueType = 'WARNING' AND vi.validationJobId =:id)" + + "GROUP BY vi.issueText, vi.recordUrl") + List getAllWarningsRecordUrls(@Param("id") int validationJobId); + +} diff --git a/src/main/java/eu/dnetlib/validatorapi/repository/ValidationJobRepository.java b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationJobRepository.java similarity index 85% rename from src/main/java/eu/dnetlib/validatorapi/repository/ValidationJobRepository.java rename to src/main/java/eu/dnetlib/validatorapi/repositories/ValidationJobRepository.java index fbbd14d..38893f1 100644 --- a/src/main/java/eu/dnetlib/validatorapi/repository/ValidationJobRepository.java +++ b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationJobRepository.java @@ -1,4 +1,4 @@ -package eu.dnetlib.validatorapi.repository; +package eu.dnetlib.validatorapi.repositories; import eu.dnetlib.validatorapi.entities.ValidationJob; import org.springframework.data.jpa.repository.JpaRepository; diff --git a/src/main/java/eu/dnetlib/validatorapi/repository/ValidationResultRepository.java b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java similarity index 60% rename from src/main/java/eu/dnetlib/validatorapi/repository/ValidationResultRepository.java rename to src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java index d4cce6f..3f33ae4 100644 --- a/src/main/java/eu/dnetlib/validatorapi/repository/ValidationResultRepository.java +++ b/src/main/java/eu/dnetlib/validatorapi/repositories/ValidationResultRepository.java @@ -1,23 +1,22 @@ -package eu.dnetlib.validatorapi.repository; +package eu.dnetlib.validatorapi.repositories; import eu.dnetlib.validatorapi.entities.SummaryResult; -import eu.dnetlib.validatorapi.entities.ValidationJobResult; +import eu.dnetlib.validatorapi.entities.ValidationRuleResult; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import java.util.List; @Repository -public interface ValidationResultRepository extends JpaRepository { +public interface ValidationResultRepository extends JpaRepository { @Query(value = "SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, sr.ruleWeight," + "COUNT(CASE WHEN sr.status = 'SUCCESS' THEN 1 END) AS passed_records, " + - "COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failed_records, " + - "COUNT(CASE WHEN array_length(sr.errors, 1) > 0 THEN 1 END) AS errors, " + - "COUNT(CASE WHEN array_length(sr.warnings, 1) > 0 THEN 1 END) AS warnings) " + - "FROM ValidationJobResult sr " + + "COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failed_records) " + + "FROM ValidationRuleResult sr " + "WHERE sr.validationJobId = :id " + "GROUP BY sr.ruleName, sr.ruleWeight") - List getSummaryResult(); + List getSummaryResult(@Param("id") int validationJobId); }