package eu.dnetlib.ariadneplus.graphdb; import java.time.LocalDateTime; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.eclipse.rdf4j.RDF4JException; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Statement; import org.eclipse.rdf4j.model.ValueFactory; import org.eclipse.rdf4j.query.*; import org.eclipse.rdf4j.repository.Repository; import org.eclipse.rdf4j.repository.RepositoryConnection; import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager; import org.eclipse.rdf4j.repository.util.Repositories; import org.eclipse.rdf4j.rio.RDFFormat; import eu.dnetlib.ariadneplus.publisher.AriadnePlusPublisherException; import eu.dnetlib.ariadneplus.rdf.RecordParserHelper; import net.sf.saxon.s9api.SaxonApiException; /** * @author enrico.ottonello * */ public class GraphDBClient { private static final Log log = LogFactory.getLog(GraphDBClient.class); public static final String PROVENANCE_NS = "http://www.d-net.research-infrastructures.eu/provenance/"; public static final int NUM_RECORDS_THRESHOLD = 10; private RecordParserHelper recordParserHelper; private String graphDBServerUrl; private String graphDBBaseURI; private String writerUser; private String writerPwd; private String repository; protected GraphDBClient(final RecordParserHelper recordParserHelper, final String graphDBServerUrl, final String graphDBBaseURI, final String writerUser, final String writerPwd, final String repository) { this.recordParserHelper = recordParserHelper; this.graphDBServerUrl = graphDBServerUrl; this.graphDBBaseURI = graphDBBaseURI; this.writerUser = writerUser; this.writerPwd = writerPwd; this.repository = repository; } public long feed(final String record) throws AriadnePlusPublisherException{ try { String objIdentifier = recordParserHelper.getObjIdentifier(record); if (StringUtils.isBlank(objIdentifier)) { log.warn("Got record with no objIdentifier -- skipping"); return 0; } log.debug("init connection to graphDBServerUrl " + this.graphDBServerUrl); RemoteRepositoryManager manager = new RemoteRepositoryManager(this.graphDBServerUrl); manager.init(); manager.setUsernameAndPassword(getWriterUser(), getWriterPwd()); log.debug("manager init"); Repository repository = manager.getRepository(getRepository()); ValueFactory factory = repository.getValueFactory(); String datasourceApi = recordParserHelper.getDatasourceApi(record); IRI graph = factory.createIRI(getGraphDBBaseURI(), datasourceApi); try (RepositoryConnection con = repository.getConnection()) { log.debug("connection established"); con.begin(); String recordURI = getRecordURI(objIdentifier, datasourceApi); log.debug("Trying to adding record with recordURI " + recordURI + " into graph " + graph); con.add(IOUtils.toInputStream(getRDFBlock(record), "UTF-8"), recordURI, RDFFormat.RDFXML, graph); con.commit(); log.debug("statement added"); con.close(); } catch (RDF4JException e) { log.error("error adding statement ...", e); } repository.shutDown(); manager.shutDown(); log.debug("manager shutDown"); return 1; }catch(Throwable e){ log.error(e); throw new AriadnePlusPublisherException(e); } } public long feedProvenance(final String datasource, final String datasourceApi) throws AriadnePlusPublisherException { try { log.debug("init connection to graphDBServerUrl " + this.graphDBServerUrl); RemoteRepositoryManager manager = new RemoteRepositoryManager(this.graphDBServerUrl); manager.init(); manager.setUsernameAndPassword(getWriterUser(), getWriterPwd()); log.debug("manager init"); Repository repository = manager.getRepository(getRepository()); ValueFactory factory = repository.getValueFactory(); IRI IS_API_OF = factory.createIRI(PROVENANCE_NS, "isApiOf"); IRI INSERTED_IN_DATE = factory.createIRI(PROVENANCE_NS, "insertedInDate"); IRI rApi = factory.createIRI(getGraphDBBaseURI(), datasourceApi); Statement stmApi = factory.createStatement(rApi, IS_API_OF, factory.createLiteral(datasource)); LocalDateTime now = LocalDateTime.now(); Statement stmInsertedDate = factory.createStatement(rApi, INSERTED_IN_DATE, factory.createLiteral(now.toString())); IRI datasourceApisGraph = factory.createIRI(getGraphDBBaseURI(), "datasourceApis"); try (RepositoryConnection con = repository.getConnection()) { con.begin(); log.debug("Adding stmt " + stmApi.toString() + " into graph " + datasourceApisGraph.toString()); con.remove(rApi, INSERTED_IN_DATE, null, datasourceApisGraph); con.add(stmApi, datasourceApisGraph); log.debug("Adding stmt " + stmInsertedDate.toString() + " into graph " + datasourceApisGraph.toString()); con.add(stmInsertedDate, datasourceApisGraph); con.commit(); log.debug("statements added"); con.close(); } catch (RDF4JException e) { log.error("error adding statement ...", e); throw new AriadnePlusPublisherException(e); } repository.shutDown(); manager.shutDown(); return 200; } catch(Throwable e){ log.error(e); throw new AriadnePlusPublisherException(e); } } public long dropDatasourceApiGraph(final String datasourceApi) throws AriadnePlusPublisherException { try { log.debug("init connection to graphDBServerUrl " + this.graphDBServerUrl); RemoteRepositoryManager manager = new RemoteRepositoryManager(this.graphDBServerUrl); manager.init(); manager.setUsernameAndPassword(getWriterUser(), getWriterPwd()); Repository repository = manager.getRepository(getRepository()); if (repository==null) { throw new AriadnePlusPublisherException("GraphDB repository not found"); } ValueFactory factory = repository.getValueFactory(); IRI rApi = factory.createIRI(getGraphDBBaseURI(), datasourceApi); try (RepositoryConnection con = repository.getConnection()) { log.debug("removing namedGraph: " + rApi); Repositories.consume(repository, conn -> conn.clear(rApi)); } catch (RDF4JException e) { log.error("error removing datasourceApi partition info ", e); throw new AriadnePlusPublisherException(e); } repository.shutDown(); manager.shutDown(); return 200; } catch(Throwable e){ log.error("error removing datasourceApi partition info ", e); throw new AriadnePlusPublisherException(e); } } private String getRecordURI(final String objIdentifier, final String datasourceApi) { return "/" + datasourceApi + "/" + objIdentifier; } public RecordParserHelper getRecordParserHelper() { return recordParserHelper; } public void setRecordParserHelper(final RecordParserHelper recordParserHelper) { this.recordParserHelper = recordParserHelper; } public void setDefaultBaseURI(final String defaultBaseURI) { this.graphDBServerUrl = defaultBaseURI; } public String getRDFBlock(final String record) throws SaxonApiException{ recordParserHelper.init(); try { if (StringUtils.isBlank(record)) { log.warn("Got empty record"); return ""; } String objIdentifier = recordParserHelper.getObjIdentifier(record); if (StringUtils.isBlank(objIdentifier)) { log.warn("Got record with no objIdentifier -- skipping"); return ""; } String rdfBlock = recordParserHelper.getRDF(record); if (StringUtils.isBlank(rdfBlock)) { log.warn("Missing rdf:RDF in record with objIdentifier " + objIdentifier); } return rdfBlock; }catch(Throwable e){ log.error(e); throw e; } } public String getGraphDBBaseURI() { return graphDBBaseURI; } public void setGraphDBBaseURI(String graphDBBaseURI) { this.graphDBBaseURI = graphDBBaseURI; } public String getWriterUser() { return writerUser; } public void setWriterUser(String writerUser) { this.writerUser = writerUser; } public String getWriterPwd() { return writerPwd; } public void setWriterPwd(String writerPwd) { this.writerPwd = writerPwd; } public String getRepository() { return repository; } public void setRepository(String repository) { this.repository = repository; } public String updateSparql(final String queryValue) throws AriadnePlusPublisherException{ try { String result = new String(""); log.debug("init connection to graphDBServerUrl " + this.graphDBServerUrl); RemoteRepositoryManager manager = new RemoteRepositoryManager(this.graphDBServerUrl); manager.init(); manager.setUsernameAndPassword(getWriterUser(), getWriterPwd()); log.debug("manager init"); Repository repository = manager.getRepository(getRepository()); ValueFactory factory = repository.getValueFactory(); try (RepositoryConnection con = repository.getConnection()) { log.debug("connection established"); con.begin(); Update updateResultQuery = con.prepareUpdate(queryValue); if (updateResultQuery!=null) { updateResultQuery.execute(); result = "updated"; } else { result = "No result."; } log.debug("query result: "+result); con.commit(); log.debug("query executed"); con.close(); } catch (RDF4JException e) { log.error("error executing query ...", e); } repository.shutDown(); manager.shutDown(); log.debug("manager shutDown"); return result; }catch(Throwable e){ log.error(e); throw new AriadnePlusPublisherException(e); } } }