D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

rdf handling move away from this node

This commit is contained in:
Enrico Ottonello 2019-12-16 15:32:20 +01:00
parent 98b452c59a
commit 5087750f96
1 changed files with 58 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ElasticSearchIndexJobNode.java
View File

@ -1,27 +1,25 @@
package eu.dnetlib.ariadneplus.workflows.nodes; package eu.dnetlib.ariadneplus.workflows.nodes;
import eu.dnetlib.ariadneplus.rdf.RecordParserHelper;
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
import eu.dnetlib.enabling.resultset.client.ResultSetClient;
import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
import eu.dnetlib.msro.workflows.graph.Arc;
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
import eu.dnetlib.msro.workflows.procs.Env;
import eu.dnetlib.rmi.common.ResultSet;
import eu.dnetlib.rmi.manager.MSROException;
import net.sf.saxon.s9api.SaxonApiException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.BufferedOutputStream; import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Collections;
import java.util.Random;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonFactory;
@ -39,17 +37,14 @@ import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.vocabulary.RDF; import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS; import com.hp.hpl.jena.vocabulary.RDFS;
import org.springframework.http.HttpEntity; import eu.dnetlib.enabling.locators.UniqueServiceLocator;
import org.springframework.http.HttpHeaders; import eu.dnetlib.enabling.resultset.client.ResultSetClient;
import org.springframework.http.HttpMethod; import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
import org.springframework.http.MediaType; import eu.dnetlib.msro.workflows.graph.Arc;
import org.springframework.http.ResponseEntity; import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
import org.springframework.http.client.ClientHttpRequestFactory; import eu.dnetlib.msro.workflows.procs.Env;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import eu.dnetlib.rmi.common.ResultSet;
import org.springframework.web.client.RestTemplate; import eu.dnetlib.rmi.manager.MSROException;
import java.util.Collections;
import java.util.Random;
public class ElasticSearchIndexJobNode extends SimpleJobNode { public class ElasticSearchIndexJobNode extends SimpleJobNode {
@ -67,7 +62,7 @@ public class ElasticSearchIndexJobNode extends SimpleJobNode {
private boolean verboseLogging; private boolean verboseLogging;
// @Autowired // @Autowired
private RecordParserHelper recordParserHelper = new RecordParserHelper(); // private RecordParserHelper recordParserHelper = new RecordParserHelper();
@Autowired @Autowired
@ -90,7 +85,8 @@ public class ElasticSearchIndexJobNode extends SimpleJobNode {
for (String record : getResultSetClient().iter(rsIn, String.class)) { for (String record : getResultSetClient().iter(rsIn, String.class)) {
// log.debug(record); // log.debug(record);
String rdfBlock = getRDFBlock(record); // String rdfBlock = getRDFBlock(record);
String rdfBlock = "";
try { try {
String jsonData = prepareJsonData(rdfBlock); String jsonData = prepareJsonData(rdfBlock);
feed(jsonData); feed(jsonData);
@ -239,31 +235,31 @@ public class ElasticSearchIndexJobNode extends SimpleJobNode {
// RDF HELPERS // RDF HELPERS
public String getRDFBlock(final String record) throws SaxonApiException{ // public String getRDFBlock(final String record) throws SaxonApiException{
recordParserHelper.init(); // recordParserHelper.init();
Model md = null ; // Model md = null ;
try { // try {
if (StringUtils.isBlank(record)) { // if (StringUtils.isBlank(record)) {
log.warn("Got empty record"); // log.warn("Got empty record");
return ""; // return "";
} // }
String objIdentifier = recordParserHelper.getObjIdentifier(record); // String objIdentifier = recordParserHelper.getObjIdentifier(record);
if (StringUtils.isBlank(objIdentifier)) { // if (StringUtils.isBlank(objIdentifier)) {
log.warn("Got record with no objIdentifier -- skipping"); // log.warn("Got record with no objIdentifier -- skipping");
return ""; // return "";
} // }
log.debug(objIdentifier); // log.debug(objIdentifier);
String rdfBlock = recordParserHelper.getRDF(record); // String rdfBlock = recordParserHelper.getRDF(record);
if (StringUtils.isBlank(rdfBlock)) { // if (StringUtils.isBlank(rdfBlock)) {
log.warn("Missing rdf:RDF in record with objIdentifier " + objIdentifier); // log.warn("Missing rdf:RDF in record with objIdentifier " + objIdentifier);
} // }
log.debug("\n\n" + rdfBlock + "\n\n" ); // log.debug("\n\n" + rdfBlock + "\n\n" );
return rdfBlock; // return rdfBlock;
}catch(Throwable e){ // }catch(Throwable e){
log.error(e); // log.error(e);
throw e; // throw e;
} // }
} // }
@ -331,12 +327,12 @@ public class ElasticSearchIndexJobNode extends SimpleJobNode {
this.resultSetClient = resultSetClient; this.resultSetClient = resultSetClient;
} }
public RecordParserHelper getRecordParserHelper() { // public RecordParserHelper getRecordParserHelper() {
return recordParserHelper; // return recordParserHelper;
} // }
//
public void setRecordParserHelper(RecordParserHelper recordParserHelper) { // public void setRecordParserHelper(RecordParserHelper recordParserHelper) {
this.recordParserHelper = recordParserHelper; // this.recordParserHelper = recordParserHelper;
} // }
} }