D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

foreach published api, the datasource name is stored into graph datasourceApis

This commit is contained in:
Enrico Ottonello 2020-01-24 10:52:04 +01:00
parent d3b4e6c864
commit 17552bafa0
6 changed files with 33 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/graphdb/GraphDBClient.java
View File

@ -196,7 +196,7 @@ public class GraphDBClient {
} }
public long feedProvenance(final String datasourceApi) throws AriadnePlusPublisherException { public long feedProvenance(final String datasource, final String datasourceApi) throws AriadnePlusPublisherException {
try { try {
log.debug("init connection to graphDBServerUrl " + this.graphDBServerUrl); log.debug("init connection to graphDBServerUrl " + this.graphDBServerUrl);
@ -209,13 +209,14 @@ public class GraphDBClient {
IRI IS_API_OF = factory.createIRI(PROVENANCE_NS, "isApiOf"); IRI IS_API_OF = factory.createIRI(PROVENANCE_NS, "isApiOf");
IRI INSERTED_IN_DATE = factory.createIRI(PROVENANCE_NS, "insertedInDate"); IRI INSERTED_IN_DATE = factory.createIRI(PROVENANCE_NS, "insertedInDate");
IRI rApi = factory.createIRI(getGraphDBBaseURI(), datasourceApi); IRI rApi = factory.createIRI(getGraphDBBaseURI(), datasourceApi);
Statement stmApi = factory.createStatement(rApi, IS_API_OF, factory.createLiteral(datasourceApi)); Statement stmApi = factory.createStatement(rApi, IS_API_OF, factory.createLiteral(datasource));
LocalDateTime now = LocalDateTime.now(); LocalDateTime now = LocalDateTime.now();
Statement stmInsertedDate = factory.createStatement(rApi, INSERTED_IN_DATE, factory.createLiteral(now.toString())); Statement stmInsertedDate = factory.createStatement(rApi, INSERTED_IN_DATE, factory.createLiteral(now.toString()));
IRI datasourceApisGraph = factory.createIRI(getGraphDBBaseURI(), "datasourceApis"); IRI datasourceApisGraph = factory.createIRI(getGraphDBBaseURI(), "datasourceApis");
try (RepositoryConnection con = repository.getConnection()) { try (RepositoryConnection con = repository.getConnection()) {
con.begin(); con.begin();
log.debug("Adding stmt " + stmApi.toString() + " into graph " + datasourceApisGraph.toString()); log.debug("Adding stmt " + stmApi.toString() + " into graph " + datasourceApisGraph.toString());
con.remove(rApi, INSERTED_IN_DATE, null, datasourceApisGraph);
con.add(stmApi, datasourceApisGraph); con.add(stmApi, datasourceApisGraph);
log.debug("Adding stmt " + stmInsertedDate.toString() + " into graph " + datasourceApisGraph.toString()); log.debug("Adding stmt " + stmInsertedDate.toString() + " into graph " + datasourceApisGraph.toString());
con.add(stmInsertedDate, datasourceApisGraph); con.add(stmInsertedDate, datasourceApisGraph);

4
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherController.java
View File

@ -39,11 +39,11 @@ public class AriadnePlusPublisherController {
} }
@RequestMapping(value = "/feedProvenance", method = RequestMethod.POST) @RequestMapping(value = "/feedProvenance", method = RequestMethod.POST)
public void feedProvenance(@RequestParam final String datasourceApi, @RequestParam(required = false) String ariadneplusTarget) throws AriadnePlusPublisherException { public void feedProvenance(@RequestParam final String datasource, @RequestParam final String datasourceApi, @RequestParam(required = false) String ariadneplusTarget) throws AriadnePlusPublisherException {
if (ariadneplusTarget==null) { if (ariadneplusTarget==null) {
ariadneplusTarget = DEFAULT_TARGET_ENDPOINT; ariadneplusTarget = DEFAULT_TARGET_ENDPOINT;
} }
getAriadnePlusPublisherHelper().feedProvenance(datasourceApi, getTarget(ariadneplusTarget)); getAriadnePlusPublisherHelper().feedProvenance(datasource, datasourceApi, getTarget(ariadneplusTarget));
} }
@RequestMapping(value = "/dropDatasourceApisPartitionInfo", method = RequestMethod.POST) @RequestMapping(value = "/dropDatasourceApisPartitionInfo", method = RequestMethod.POST)

15
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/publisher/AriadnePlusPublisherHelper.java
View File

@ -35,10 +35,10 @@ public class AriadnePlusPublisherHelper {
} }
public void feedProvenance(final String datasourceApi, final AriadnePlusTargets target) throws AriadnePlusPublisherException { public void feedProvenance(final String datasource, final String datasourceApi, final AriadnePlusTargets target) throws AriadnePlusPublisherException {
switch(target){ switch(target){
case GRAPHDB: case GRAPHDB:
feedProvenance(datasourceApi); feedProvenance(datasource, datasourceApi);
break; break;
default: throw new AriadnePlusPublisherException("Target "+target+" not supported yet"); default: throw new AriadnePlusPublisherException("Target "+target+" not supported yet");
} }
@ -72,10 +72,10 @@ public class AriadnePlusPublisherHelper {
graphDBClient.feed(record); graphDBClient.feed(record);
} }
private void feedProvenance(final String datasourceApi) throws AriadnePlusPublisherException { private void feedProvenance(final String datasource, final String datasourceApi) throws AriadnePlusPublisherException {
log.debug("Feed Provenance " + datasourceApi); log.debug("Feed Provenance " + datasource + " - " + datasourceApi);
GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient();
graphDBClient.feedProvenance(datasourceApi); graphDBClient.feedProvenance(datasource, datasourceApi);
} }
private void dropDatasourceApisPartitionInfo(final String datasourceApi) throws AriadnePlusPublisherException { private void dropDatasourceApisPartitionInfo(final String datasourceApi) throws AriadnePlusPublisherException {
@ -87,8 +87,7 @@ public class AriadnePlusPublisherHelper {
private long unpublishGraphDB(final String datasourceInterface) { private long unpublishGraphDB(final String datasourceInterface) {
log.info("Unpublishing from graphdb "+datasourceInterface); log.info("Unpublishing from graphdb "+datasourceInterface);
GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient(); GraphDBClient graphDBClient = this.graphdbClientFactory.getGraphDBClient();
long deletedTriples = graphDBClient.drop(datasourceInterface);
log.info("# triples deleted for "+datasourceInterface+": "+deletedTriples); return 0;
return deletedTriples;
} }
} }

14
dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/PublishGraphDBJobNode.java
View File

@ -46,6 +46,7 @@ public class PublishGraphDBJobNode extends AsyncJobNode {
private String publisherEndpoint; private String publisherEndpoint;
private String datasourceInterface; private String datasourceInterface;
private String datasource;
//for parallel requests to the publisher endpoint //for parallel requests to the publisher endpoint
private int nThreads = 5; private int nThreads = 5;
@ -178,6 +179,9 @@ public class PublishGraphDBJobNode extends AsyncJobNode {
String datasourceInterfaceValue = getDatasourceInterface(); String datasourceInterfaceValue = getDatasourceInterface();
log.info("feeding provenance for datasourceInterface " + datasourceInterfaceValue); log.info("feeding provenance for datasourceInterface " + datasourceInterfaceValue);
params.add(new BasicNameValuePair("datasourceApi", datasourceInterfaceValue)); params.add(new BasicNameValuePair("datasourceApi", datasourceInterfaceValue));
String datasourceValue = getDatasource();
log.info("feeding provenance for datasource " + datasourceValue);
params.add(new BasicNameValuePair("datasource", datasourceValue));
UrlEncodedFormEntity ent = new UrlEncodedFormEntity(params, "UTF-8"); UrlEncodedFormEntity ent = new UrlEncodedFormEntity(params, "UTF-8");
post.setEntity(ent); post.setEntity(ent);
responsePFPOST = client.execute(post); responsePFPOST = client.execute(post);
@ -256,4 +260,14 @@ public class PublishGraphDBJobNode extends AsyncJobNode {
token.setProgressProvider(new ResultsetProgressProvider(token.getEnv().getAttribute(getEprParam(), ResultSet.class), this.resultSetClient)); token.setProgressProvider(new ResultsetProgressProvider(token.getEnv().getAttribute(getEprParam(), ResultSet.class), this.resultSetClient));
} }
public String getDatasource() {
return datasource;
}
public void setDatasource(String datasource) {
this.datasource = datasource;
}
} }

7
dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/x3m_aggregation_wf.xml.st
View File

@ -105,9 +105,10 @@
<PARAM name="wfTemplateId" value="7426eaaf-93c9-4914-b69a-c9d5c478405a_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/> <PARAM name="wfTemplateId" value="7426eaaf-93c9-4914-b69a-c9d5c478405a_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
<PARAM name="wfTemplateParams"> <PARAM name="wfTemplateParams">
<MAP> <MAP>
<ENTRY key="dsId" value="d52b32f0-2b19-4bef-930d-470ef044def5_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU="/> <ENTRY key="dsId" value="$dsId$" />
<ENTRY key="interface" value="api_________::ariadne_plus::ads::1"/> <ENTRY key="dsName" value="$dsName$" />
<ENTRY key="cleanMdstoreId" ref="cleanMdstoreId"/> <ENTRY key="interface" value="$interface$" />
<ENTRY key="cleanMdstoreId" ref="cleanMdstoreId"/>
<ENTRY key="publisherEndpoint" ref="publisherEndpoint" /> <ENTRY key="publisherEndpoint" ref="publisherEndpoint" />
</MAP> </MAP>
</PARAM> </PARAM>

4
dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/graphdb_template.xml
View File

@ -9,10 +9,11 @@
<BODY> <BODY>
<CONFIGURATION> <CONFIGURATION>
<PARAMETERS> <PARAMETERS>
<PARAM description="Datasource Name" name="dsName" required="true" type="string"/>
<PARAM description="Datasource Id" name="dsId" required="true" type="string"/> <PARAM description="Datasource Id" name="dsId" required="true" type="string"/>
<PARAM description="Datasource Interface" name="interface" required="true" type="string"/> <PARAM description="Datasource Interface" name="interface" required="true" type="string"/>
<PARAM description="Store for transformed records" name="cleanMdstoreId" required="true" type="string"/> <PARAM description="Store for transformed records" name="cleanMdstoreId" required="true" type="string"/>
<PARAM name="publisherEndpoint" description="AriadnePlus Publisher Endpoint" required="true" managedBy="user" type="string">http://localhost:8080/ariadneplus/publish</PARAM> <PARAM name="publisherEndpoint" description="AriadnePlus Publisher Endpoint" required="true" type="string">http://localhost:8080/ariadneplus/publish</PARAM>
</PARAMETERS> </PARAMETERS>
<WORKFLOW> <WORKFLOW>
<NODE isStart="true" name="fetchMdStore" type="FetchMDStoreRecords"> <NODE isStart="true" name="fetchMdStore" type="FetchMDStoreRecords">
@ -31,6 +32,7 @@
<PARAM name="eprParam" value="clean_epr"/> <PARAM name="eprParam" value="clean_epr"/>
<PARAM name="publisherEndpoint" ref="publisherEndpoint"/> <PARAM name="publisherEndpoint" ref="publisherEndpoint"/>
<PARAM name="datasourceInterface" ref="interface"/> <PARAM name="datasourceInterface" ref="interface"/>
<PARAM name="datasource" ref="dsName"/>
</PARAMETERS> </PARAMETERS>
<ARCS> <ARCS>
<ARC to="success"/> <ARC to="success"/>