D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zeppelin: remove expired notes

This commit is contained in:
Michele Artini 2021-11-02 11:50:50 +01:00
parent a7f8f8f061
commit 60629198bd
3 changed files with 34 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/MainApplication.java
View File

@ -5,6 +5,7 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.scheduling.annotation.EnableScheduling;
import eu.dnetlib.common.app.AbstractDnetApp;
import springfox.documentation.builders.ApiInfoBuilder;
@ -16,6 +17,7 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2;
@SpringBootApplication
@EnableSwagger2
@EnableCaching
@EnableScheduling
@EntityScan("eu.dnetlib.dhp.schema.mdstore")
public class MainApplication extends AbstractDnetApp {

8
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java
View File

@ -16,15 +16,15 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import eu.dnetlib.dhp.schema.mdstore.MDStore;
import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreCurrentVersionRepository;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreRepository;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreVersionRepository;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreWithInfoRepository;
import eu.dnetlib.dhp.schema.mdstore.MDStore;
import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
@Service
public class DatabaseUtils {

44
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java
View File

@ -4,6 +4,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -26,6 +27,7 @@ import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
@ -59,6 +61,8 @@ public class ZeppelinClient {
private static final Map<String, List<String>> DEFAULT_RIGHTS = new LinkedHashMap<>();
private static final Integer MAX_NUMBER_OF_MD_NOTES = 2;
@PostConstruct
public void init() {
DEFAULT_RIGHTS.put("owners", Arrays.asList(zeppelinLogin));
@ -118,8 +122,8 @@ public class ZeppelinClient {
private String cloneNote(final String noteId, final String newName, final MDStoreWithInfo mdstore, final String currentVersionPath)
throws MDStoreManagerException {
final String newId = callApi(HttpMethod.POST, "notebook/" + noteId, StringResponse.class, new Note(newName)).getBody();
callApi(HttpMethod.POST, "notebook/" + noteId + "/paragraph", StringResponse.class, confParagraph(mdstore, currentVersionPath)).getBody();
callApi(HttpMethod.PUT, "notebook/" + noteId + "/permissions", SimpleResponse.class, DEFAULT_RIGHTS);
callApi(HttpMethod.POST, "notebook/" + newId + "/paragraph", StringResponse.class, confParagraph(mdstore, currentVersionPath)).getBody();
callApi(HttpMethod.PUT, "notebook/" + newId + "/permissions", SimpleResponse.class, DEFAULT_RIGHTS);
log.info("New note created, id: " + newId + ", name: " + newName);
@ -143,13 +147,30 @@ public class ZeppelinClient {
}
}
// TODO: prepare the cron job
@Scheduled(fixedRate = 12 * 60 * 60 * 1000) // 12 hours
public void cleanExpiredNotes() {
try {
for (final Map<String, String> n : listNotes()) {
final String id = n.get("id");
if (n.get("name").startsWith(zeppelinNamePrefix + "/notes/") && isExpired(id)) {
deleteNote(id);
// I sort the notes according to the version datestamp (more recent first)
final List<Map<String, String>> notes = listNotes()
.stream()
.filter(n -> n.get("name").startsWith(zeppelinNamePrefix + "/notes/"))
.sorted((o1, o2) -> StringUtils.compare(o2.get("name"), o1.get("name")))
.collect(Collectors.toList());
final Map<String, Integer> map = new HashMap<>();
for (final Map<String, String> n : notes) {
final String firstPart = StringUtils.substringBeforeLast(n.get("name"), "-");
if (!map.containsKey(firstPart)) {
log.info("Evaluating note " + n.get("name") + " for deletion: CONFIRMED");
map.put(firstPart, 1);
} else if (map.get(firstPart) < MAX_NUMBER_OF_MD_NOTES) {
log.info("Evaluating note " + n.get("name") + " for deletion: CONFIRMED");
map.put(firstPart, map.get(firstPart) + 1);
} else {
log.info("Evaluating note " + n.get("name") + " for deletion: TO_DELETE");
callApi(HttpMethod.DELETE, "notebook/" + n.get("id"), SimpleResponse.class, null);
}
}
} catch (final Exception e) {
@ -157,15 +178,6 @@ public class ZeppelinClient {
}
}
private void deleteNote(final String noteId) {
callApi(HttpMethod.DELETE, "notebook/" + noteId, SimpleResponse.class, null);
}
private boolean isExpired(final String id) {
// TODO Auto-generated method stub
return false;
}
private <T extends HasStatus> T callApi(final HttpMethod method, final String api, final Class<T> resClazz, final Object objRequest) {
if (jsessionid == null) {