From 60629198bd6431343bda2d3b7745d25035b84b06 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 2 Nov 2021 11:50:50 +0100 Subject: [PATCH] zeppelin: remove expired notes --- .../data/mdstore/manager/MainApplication.java | 2 + .../mdstore/manager/utils/DatabaseUtils.java | 8 ++-- .../mdstore/manager/utils/ZeppelinClient.java | 44 ++++++++++++------- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/MainApplication.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/MainApplication.java index d5e4d504..7a2f6de8 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/MainApplication.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/MainApplication.java @@ -5,6 +5,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.cache.annotation.EnableCaching; +import org.springframework.scheduling.annotation.EnableScheduling; import eu.dnetlib.common.app.AbstractDnetApp; import springfox.documentation.builders.ApiInfoBuilder; @@ -16,6 +17,7 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2; @SpringBootApplication @EnableSwagger2 @EnableCaching +@EnableScheduling @EntityScan("eu.dnetlib.dhp.schema.mdstore") public class MainApplication extends AbstractDnetApp { diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java index c5f1551d..4a2313e5 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java @@ -16,15 +16,15 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; -import eu.dnetlib.dhp.schema.mdstore.MDStore; -import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion; -import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; -import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo; import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException; import eu.dnetlib.data.mdstore.manager.repository.MDStoreCurrentVersionRepository; import eu.dnetlib.data.mdstore.manager.repository.MDStoreRepository; import eu.dnetlib.data.mdstore.manager.repository.MDStoreVersionRepository; import eu.dnetlib.data.mdstore.manager.repository.MDStoreWithInfoRepository; +import eu.dnetlib.dhp.schema.mdstore.MDStore; +import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; +import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo; @Service public class DatabaseUtils { diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java index 98e864ce..dc77620a 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -26,6 +27,7 @@ import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; @@ -59,6 +61,8 @@ public class ZeppelinClient { private static final Map> DEFAULT_RIGHTS = new LinkedHashMap<>(); + private static final Integer MAX_NUMBER_OF_MD_NOTES = 2; + @PostConstruct public void init() { DEFAULT_RIGHTS.put("owners", Arrays.asList(zeppelinLogin)); @@ -118,8 +122,8 @@ public class ZeppelinClient { private String cloneNote(final String noteId, final String newName, final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException { final String newId = callApi(HttpMethod.POST, "notebook/" + noteId, StringResponse.class, new Note(newName)).getBody(); - callApi(HttpMethod.POST, "notebook/" + noteId + "/paragraph", StringResponse.class, confParagraph(mdstore, currentVersionPath)).getBody(); - callApi(HttpMethod.PUT, "notebook/" + noteId + "/permissions", SimpleResponse.class, DEFAULT_RIGHTS); + callApi(HttpMethod.POST, "notebook/" + newId + "/paragraph", StringResponse.class, confParagraph(mdstore, currentVersionPath)).getBody(); + callApi(HttpMethod.PUT, "notebook/" + newId + "/permissions", SimpleResponse.class, DEFAULT_RIGHTS); log.info("New note created, id: " + newId + ", name: " + newName); @@ -143,13 +147,30 @@ public class ZeppelinClient { } } - // TODO: prepare the cron job + @Scheduled(fixedRate = 12 * 60 * 60 * 1000) // 12 hours public void cleanExpiredNotes() { try { - for (final Map n : listNotes()) { - final String id = n.get("id"); - if (n.get("name").startsWith(zeppelinNamePrefix + "/notes/") && isExpired(id)) { - deleteNote(id); + + // I sort the notes according to the version datestamp (more recent first) + final List> notes = listNotes() + .stream() + .filter(n -> n.get("name").startsWith(zeppelinNamePrefix + "/notes/")) + .sorted((o1, o2) -> StringUtils.compare(o2.get("name"), o1.get("name"))) + .collect(Collectors.toList()); + + final Map map = new HashMap<>(); + for (final Map n : notes) { + + final String firstPart = StringUtils.substringBeforeLast(n.get("name"), "-"); + if (!map.containsKey(firstPart)) { + log.info("Evaluating note " + n.get("name") + " for deletion: CONFIRMED"); + map.put(firstPart, 1); + } else if (map.get(firstPart) < MAX_NUMBER_OF_MD_NOTES) { + log.info("Evaluating note " + n.get("name") + " for deletion: CONFIRMED"); + map.put(firstPart, map.get(firstPart) + 1); + } else { + log.info("Evaluating note " + n.get("name") + " for deletion: TO_DELETE"); + callApi(HttpMethod.DELETE, "notebook/" + n.get("id"), SimpleResponse.class, null); } } } catch (final Exception e) { @@ -157,15 +178,6 @@ public class ZeppelinClient { } } - private void deleteNote(final String noteId) { - callApi(HttpMethod.DELETE, "notebook/" + noteId, SimpleResponse.class, null); - } - - private boolean isExpired(final String id) { - // TODO Auto-generated method stub - return false; - } - private T callApi(final HttpMethod method, final String api, final Class resClazz, final Object objRequest) { if (jsessionid == null) {