D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

rdf handling move away from this node

This commit is contained in:
Enrico Ottonello 2019-12-16 15:32:20 +01:00
parent 98b452c59a
commit 5087750f96
1 changed files with 58 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ElasticSearchIndexJobNode.java
View File

@ -1,27 +1,25 @@
package eu.dnetlib.ariadneplus.workflows.nodes;
import eu.dnetlib.ariadneplus.rdf.RecordParserHelper;
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
import eu.dnetlib.enabling.resultset.client.ResultSetClient;
import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
import eu.dnetlib.msro.workflows.graph.Arc;
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
import eu.dnetlib.msro.workflows.procs.Env;
import eu.dnetlib.rmi.common.ResultSet;
import eu.dnetlib.rmi.manager.MSROException;
import net.sf.saxon.s9api.SaxonApiException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.Random;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
@ -39,17 +37,14 @@ import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import java.util.Collections;
import java.util.Random;
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
import eu.dnetlib.enabling.resultset.client.ResultSetClient;
import eu.dnetlib.enabling.resultset.factory.ResultSetFactory;
import eu.dnetlib.msro.workflows.graph.Arc;
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
import eu.dnetlib.msro.workflows.procs.Env;
import eu.dnetlib.rmi.common.ResultSet;
import eu.dnetlib.rmi.manager.MSROException;
public class ElasticSearchIndexJobNode extends SimpleJobNode {
@ -67,7 +62,7 @@ public class ElasticSearchIndexJobNode extends SimpleJobNode {
private boolean verboseLogging;
// @Autowired
private RecordParserHelper recordParserHelper = new RecordParserHelper();
// private RecordParserHelper recordParserHelper = new RecordParserHelper();
@Autowired
@ -90,7 +85,8 @@ public class ElasticSearchIndexJobNode extends SimpleJobNode {
for (String record : getResultSetClient().iter(rsIn, String.class)) {
// log.debug(record);
String rdfBlock = getRDFBlock(record);
// String rdfBlock = getRDFBlock(record);
String rdfBlock = "";
try {
String jsonData = prepareJsonData(rdfBlock);
feed(jsonData);
@ -239,31 +235,31 @@ public class ElasticSearchIndexJobNode extends SimpleJobNode {
// RDF HELPERS
public String getRDFBlock(final String record) throws SaxonApiException{
recordParserHelper.init();
Model md = null ;
try {
if (StringUtils.isBlank(record)) {
log.warn("Got empty record");
return "";
}
String objIdentifier = recordParserHelper.getObjIdentifier(record);
if (StringUtils.isBlank(objIdentifier)) {
log.warn("Got record with no objIdentifier -- skipping");
return "";
}
log.debug(objIdentifier);
String rdfBlock = recordParserHelper.getRDF(record);
if (StringUtils.isBlank(rdfBlock)) {
log.warn("Missing rdf:RDF in record with objIdentifier " + objIdentifier);
}
log.debug("\n\n" + rdfBlock + "\n\n" );
return rdfBlock;
}catch(Throwable e){
log.error(e);
throw e;
}
}
// public String getRDFBlock(final String record) throws SaxonApiException{
// recordParserHelper.init();
// Model md = null ;
// try {
// if (StringUtils.isBlank(record)) {
// log.warn("Got empty record");
// return "";
// }
// String objIdentifier = recordParserHelper.getObjIdentifier(record);
// if (StringUtils.isBlank(objIdentifier)) {
// log.warn("Got record with no objIdentifier -- skipping");
// return "";
// }
// log.debug(objIdentifier);
// String rdfBlock = recordParserHelper.getRDF(record);
// if (StringUtils.isBlank(rdfBlock)) {
// log.warn("Missing rdf:RDF in record with objIdentifier " + objIdentifier);
// }
// log.debug("\n\n" + rdfBlock + "\n\n" );
// return rdfBlock;
// }catch(Throwable e){
// log.error(e);
// throw e;
// }
// }
@ -331,12 +327,12 @@ public class ElasticSearchIndexJobNode extends SimpleJobNode {
this.resultSetClient = resultSetClient;
}
public RecordParserHelper getRecordParserHelper() {
return recordParserHelper;
}
public void setRecordParserHelper(RecordParserHelper recordParserHelper) {
this.recordParserHelper = recordParserHelper;
}
// public RecordParserHelper getRecordParserHelper() {
// return recordParserHelper;
// }
//
// public void setRecordParserHelper(RecordParserHelper recordParserHelper) {
// this.recordParserHelper = recordParserHelper;
// }
}