miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

method to purge all items in the d4science catalog

This commit is contained in:
Alessia Bardi 2020-06-22 19:25:25 +02:00
parent d9c07eb800
commit a27b93859e
2 changed files with 59 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/gcat/GCatAPIClient.java
View File

@ -43,10 +43,9 @@ public class GCatAPIClient {
}
/**
* Publish the json as in the D4science catalogue as an item. TODO: does the POST returns the whole item or just its
* catalogue identifier?
* Publish the json as in the D4science catalogue as an item.
*
* @param jsonMetadata
* @param jsonMetadata the whole published json record
* @return the HTTP status code of the request
* @throws IOException
*/
@ -61,8 +60,10 @@ public class GCatAPIClient {
StringEntity entity = new StringEntity(jsonMetadata, StandardCharsets.UTF_8);
post.setEntity(entity);
HttpResponse response = client.execute(post);
System.out.println(response.getStatusLine());
System.out.println(IOUtils.toString(response.getEntity().getContent()));
if (log.isDebugEnabled()) {
log.debug(response.getStatusLine());
log.debug(IOUtils.toString(response.getEntity().getContent()));
}
return response.getStatusLine().getStatusCode();
}
}
@ -72,7 +73,7 @@ public class GCatAPIClient {
*
* @param offset offset
* @param limit limit
* @return list of json items
* @return list of catalogue item names
* @throws IOException
* @throws URISyntaxException
*/
@ -101,7 +102,6 @@ public class GCatAPIClient {
}
};
String responseBody = client.execute(get, responseHandler);
System.out.println(responseBody);
Gson gson = new Gson();
return gson.fromJson(responseBody, List.class);
}
@ -117,11 +117,34 @@ public class GCatAPIClient {
del.addHeader("Content-Type", "application/json");
del.addHeader("Accept", "application/json");
HttpResponse response = client.execute(del);
System.out.println(response.getStatusLine());
if (log.isDebugEnabled()) {
log.debug(response.getStatusLine());
}
return response.getStatusLine().getStatusCode() == HttpStatus.SC_NO_CONTENT;
}
}
public int purgeAll() throws IOException, URISyntaxException {
int count = 0;
int deleted = 0;
int failed = 0;
List<String> list = list(0, BULK_SIZE);
do {
for (String itemName : list) {
count++;
if (purge(itemName))
deleted++;
else {
failed++;
log.warn("Deletion of item " + itemName + " failed");
}
}
list = list(0, BULK_SIZE);
} while (list.size() > 0);
log.info(String.format("PurgeAll completed: total = %d; deleted = %d; failed = %d", count, deleted, failed));
return deleted;
}
protected String getCatalogueNameFrom(final String objIdentifier) {
return objIdentifier.replaceAll("::", "_");
}

28
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/GCatAPIClientTest.java
View File

@ -13,6 +13,10 @@ import org.junit.jupiter.api.Test;
import eu.dnetlib.dhp.oa.graph.dump.gcat.GCatAPIClient;
import sun.jvm.hotspot.utilities.Assert;
/**
* NEVER EVER ENABLE THIS CLASS UNLESS YOU ABSOLUTELY KNOW WHAT YOU ARE DOING: with the proper parameters set it can
* dropped a D4Science Catalogue
*/
@Disabled
public class GCatAPIClientTest {
@ -39,12 +43,14 @@ public class GCatAPIClientTest {
String json = IOUtils
.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/dump/gcat/gcat_pub.json"));
Assert.that(client.publish(json) == HttpStatus.SC_CREATED, "Item not created");
System.out.println("item created, now listing...");
Assert.that(client.list(0, 10).size() == 1, "List has more elements than expected");
// and then drop it
Assert
.that(
client.purge(objidentifier),
"It should return true! I just created item with name from " + objidentifier);
System.out.println("item purged");
}
@Test
@ -52,4 +58,26 @@ public class GCatAPIClientTest {
String id = "1234";
Assert.that(!client.purge(id), "It should return false! The item does not exist");
}
@Test
public void testPurgeAllEmptyCat() throws IOException, URISyntaxException {
Assert
.that(
0 == client.purgeAll(),
"Expected 0 elements in the catalogue...we dropped anything that was there...sorry");
}
@Test
public void testPublishAndPurgeAll() throws IOException, URISyntaxException {
String json = IOUtils
.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/dump/gcat/gcat_pub.json"));
Assert.that(client.publish(json) == HttpStatus.SC_CREATED, "Item not created");
System.out.println("item created, now listing...");
Assert.that(client.list(0, 10).size() == 1, "List has more elements than expected");
// and then drop all
Assert
.that(
1 == client.purgeAll(),
"Expected 1 elements in the catalogue...we dropped anything that was there...sorry");
}
}