michele.artini
/
dnet-docker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

clients

This commit is contained in:
Michele Artini 2023-10-17 14:34:28 +02:00
parent 1ac0a4247e
commit 278d719057
10 changed files with 92 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
dnet-app/apps/dnet-datasource-manager/src/main/java/eu/dnetlib/services/dsm/controller/DsmApiControllerV1.java
View File

@ -273,4 +273,28 @@ public class DsmApiControllerV1 extends AbstractDsmController {
log.info("API saved, id: " + api.getId());
}
@PostMapping("/ds/api/last-collection")
public void updateApiCollectionInfo(@RequestParam final String apiId,
@RequestParam final String mdId,
@RequestParam final long size) throws DsmException {
dsmService.updateApiCollectionInfo(apiId, mdId, size);
}
@PostMapping("/ds/api/last-aggregation")
public void updateApiAggregationInfo(@RequestParam final String apiId,
@RequestParam final String mdId,
@RequestParam final long size) throws DsmException {
dsmService.updateApiAggregationInfo(apiId, mdId, size);
}
@PostMapping("/ds/api/last-download")
public void updateApiDownloadInfo(@RequestParam final String apiId,
@RequestParam final String objId,
@RequestParam final long size) throws DsmException {
dsmService.updateApiDownloadInfo(apiId, objId, size);
}
}

6
dnet-app/apps/dnet-datasource-manager/src/main/java/eu/dnetlib/services/dsm/service/DsmService.java
View File

@ -478,7 +478,7 @@ public class DsmService {
return apiRepository.findById(id).orElseThrow(() -> new DsmException("Api not found. ID: " + id));
}
public Api updateUpdateApiCollectionInfo(final String apiId, final String mdId, final long total) throws DsmException {
public Api updateApiCollectionInfo(final String apiId, final String mdId, final long total) throws DsmException {
if (StringUtils.isNotBlank(mdId)) {
apiRepository.updateLastCollectionInfo(apiId, mdId, new Timestamp(System.currentTimeMillis()), total);
} else {
@ -487,7 +487,7 @@ public class DsmService {
return findApi(apiId);
}
public Api updateUpdateApiAggregationInfo(final String apiId, final String mdId, final long total) throws DsmException {
public Api updateApiAggregationInfo(final String apiId, final String mdId, final long total) throws DsmException {
if (StringUtils.isNotBlank(mdId)) {
apiRepository.updateLastAggregationInfo(apiId, mdId, new Timestamp(System.currentTimeMillis()), total);
} else {
@ -496,7 +496,7 @@ public class DsmService {
return findApi(apiId);
}
public Api updateUpdateApiDownloadInfo(final String apiId, final String mdId, final long total) throws DsmException {
public Api updateApiDownloadInfo(final String apiId, final String mdId, final long total) throws DsmException {
if (StringUtils.isNotBlank(mdId)) {
apiRepository.updateLastDownloadInfo(apiId, mdId, new Timestamp(System.currentTimeMillis()), total);
} else {

2
dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCleanerJobNode.java
View File

@ -80,7 +80,7 @@ public class MdCleanerJobNode extends AbstractJobNode {
})
.forEach(record -> mdstoreService.addRecord(outputMdId, record));
clientFactory.getClient(DsmClient.class).updateUpdateApiAggregationInfo(api.getId(), outputMdId, mdstoreService.getSize(outputMdId));
clientFactory.getClient(DsmClient.class).updateApiAggregationInfo(api.getId(), outputMdId, mdstoreService.getSize(outputMdId));
}

2
dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectIncrementalJobNode.java
View File

@ -74,7 +74,7 @@ public class MdCollectIncrementalJobNode extends AbstractJobNode {
.map(mdBuilder)
.forEach(record -> mdstoreService.addRecord(mdId, record));
clientFactory.getClient(DsmClient.class).updateUpdateApiCollectionInfo(api.getId(), mdId, mdstoreService.getSize(mdId));
clientFactory.getClient(DsmClient.class).updateApiCollectionInfo(api.getId(), mdId, mdstoreService.getSize(mdId));
}
private LocalDateTime findLastCollDate(final Api api) {

2
dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectRefreshJobNode.java
View File

@ -66,7 +66,7 @@ public class MdCollectRefreshJobNode extends AbstractJobNode {
.map(mdBuilder)
.forEach(record -> mdstoreService.addRecord(mdId, record));
clientFactory.getClient(DsmClient.class).updateUpdateApiCollectionInfo(api.getId(), mdId, mdstoreService.getSize(mdId));
clientFactory.getClient(DsmClient.class).updateApiCollectionInfo(api.getId(), mdId, mdstoreService.getSize(mdId));
}
}

2
dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java
View File

@ -84,7 +84,7 @@ public class MdTransformJobNode extends AbstractJobNode {
})
.forEach(record -> mdstoreService.addRecord(outputMdId, record));
clientFactory.getClient(DsmClient.class).updateUpdateApiAggregationInfo(api.getId(), outputMdId, mdstoreService.getSize(outputMdId));
clientFactory.getClient(DsmClient.class).updateApiAggregationInfo(api.getId(), outputMdId, mdstoreService.getSize(outputMdId));
}

25
dnet-app/libs/dnet-app-common/src/main/java/eu/dnetlib/common/clients/BaseMDStoreClient.java
View File

@ -1,24 +1,37 @@
package eu.dnetlib.common.clients;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import eu.dnetlib.domain.mdstore.MDStoreType;
import eu.dnetlib.domain.mdstore.MDStoreWithInfo;
public class BaseMDStoreClient extends DnetServiceClient {
public String createMDStore(final String format,
public MDStoreWithInfo createMDStore(final String format,
final String layout,
final String interpretation,
final MDStoreType type,
final String officialname,
final String dsName,
final String dsId,
final String apiId) {
// TODO (HIGH PRIORITY) Auto-generated method stub
return null;
final String url = String.format("/api/mdstores/new/%s/%s/%s/%s", URLEncoder.encode(format, StandardCharsets.UTF_8), URLEncoder
.encode(layout, StandardCharsets.UTF_8), URLEncoder
.encode(interpretation, StandardCharsets.UTF_8), URLEncoder.encode(type.name(), StandardCharsets.UTF_8));
final Map<String, Object> params = new HashMap<>();
params.put("dsName", dsName);
params.put("dsId", dsId);
params.put("apiId", apiId);
return httpPostParams(url, params, MDStoreWithInfo.class);
}
public void deleteMdStore(final String mdId) {
// TODO (HIGH PRIORITY) Auto-generated method stub
httpDelete("/api/mdstores/mdstore/" + URLEncoder.encode(mdId, StandardCharsets.UTF_8));
}
}

47
dnet-app/libs/dnet-app-common/src/main/java/eu/dnetlib/common/clients/DsmClient.java
View File

@ -1,28 +1,51 @@
package eu.dnetlib.common.clients;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import eu.dnetlib.domain.dsm.Api;
import eu.dnetlib.domain.dsm.Datasource;
import eu.dnetlib.domain.protocol.ProtocolDesc;
public class DsmClient extends DnetServiceClient {
public Iterable<?> listProtocols() {
// TODO (HIGH PRIORITY) Auto-generated method stub
return null;
public List<ProtocolDesc> listProtocols() {
return Arrays.asList(httpGet("/api/protocols/", ProtocolDesc[].class));
}
public Api updateUpdateApiCollectionInfo(final String apiId, final String mdId, final long size) {
// TODO (HIGH PRIORITY) Auto-generated method stub
return null;
public Api updateApiCollectionInfo(final String apiId, final String mdId, final long size) {
final Map<String, Object> params = new HashMap<>();
params.put("apiId", apiId);
params.put("mdId", mdId);
params.put("size", size);
httpPostParams("/api/1.0/ds/api/last-collection", params, Void.class);
return findApi(apiId);
}
public Api updateUpdateApiAggregationInfo(final String apiId, final String mdId, final long size) {
// TODO (HIGH PRIORITY) Auto-generated method stub
return null;
public Api updateApiAggregationInfo(final String apiId, final String mdId, final long size) {
final Map<String, Object> params = new HashMap<>();
params.put("apiId", apiId);
params.put("mdId", mdId);
params.put("size", size);
httpPostParams("/api/1.0/ds/api/last-aggregation", params, Void.class);
return findApi(apiId);
}
public Api updateUpdateApiDownloadInfo(final String apiId, final String objId, final long size) {
// TODO (HIGH PRIORITY) Auto-generated method stub
return null;
public Api updateApiDownloadInfo(final String apiId, final String objId, final long size) {
final Map<String, Object> params = new HashMap<>();
params.put("apiId", apiId);
params.put("objId", objId);
params.put("size", size);
httpPostParams("/api/1.0/ds/api/last-download", params, Void.class);
return findApi(apiId);
}
public Datasource findDs(final String dsId) {

6
dnet-app/libs/dnet-wf-common-nodes/src/main/java/eu/dnetlib/wfs/nodes/ds/ClearApiExtraFieldsNode.java
View File

@ -28,14 +28,14 @@ public class ClearApiExtraFieldsNode extends AbstractJobNode {
switch (infoType.toUpperCase()) {
case "COLLECT":
dsm.updateUpdateApiCollectionInfo(apiId, null, 0);
dsm.updateApiCollectionInfo(apiId, null, 0);
break;
case "AGGREGATOR":
case "TRANSFORM":
dsm.updateUpdateApiAggregationInfo(apiId, null, 0);
dsm.updateApiAggregationInfo(apiId, null, 0);
break;
case "DOWNLOAD":
dsm.updateUpdateApiDownloadInfo(apiId, null, 0);
dsm.updateApiDownloadInfo(apiId, null, 0);
break;
default:
throw new RuntimeException("Invalid infoType: " + infoType);

5
dnet-app/libs/dnet-wf-common-nodes/src/main/java/eu/dnetlib/wfs/nodes/store/CreateMdStoreNode.java
View File

@ -7,6 +7,7 @@ import eu.dnetlib.common.clients.DnetServiceClientFactory;
import eu.dnetlib.domain.dsm.Api;
import eu.dnetlib.domain.dsm.Datasource;
import eu.dnetlib.domain.mdstore.MDStoreType;
import eu.dnetlib.domain.mdstore.MDStoreWithInfo;
import eu.dnetlib.wfs.annotations.WfInputParam;
import eu.dnetlib.wfs.annotations.WfNode;
import eu.dnetlib.wfs.annotations.WfOutputParam;
@ -41,8 +42,10 @@ public class CreateMdStoreNode extends AbstractJobNode {
@Override
protected void execute() throws Exception {
mdId = clientFactory.getClient(BaseMDStoreClient.class)
final MDStoreWithInfo mdstore = clientFactory.getClient(BaseMDStoreClient.class)
.createMDStore(format, layout, interpretation, MDStoreType.valueOf(backend), ds.getOfficialname(), ds.getId(), api.getId());
mdId = mdstore.getId();
}
}