gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Migrating from HomeLibrary to StorageHub refs #17128

This commit is contained in:
Luca Frosini 2019-07-15 17:43:18 +02:00
parent 1a8c8f864a
commit 2977a684fd
3 changed files with 49 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pom.xml
View File

@ -95,15 +95,11 @@
<artifactId>google-http-client-gson</artifactId>
<version>1.21.0</version>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library-jcr</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
<artifactId>storagehub-client-library</artifactId>
<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
@ -113,7 +109,7 @@
<scope>compile</scope>
</dependency>
<!-- Dependencies force to provided -->
<!-- Dependencies forced to provided -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@ -154,7 +150,7 @@
<artifactId>common-scope-maps</artifactId>
<scope>provided</scope>
</dependency>
<!-- END Dependencies force to provided -->
<!-- END Dependencies forced to provided -->
<dependency>
<groupId>org.gcube.resources.discovery</groupId>

85
src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java
View File

@ -7,18 +7,17 @@ import java.util.Date;
import java.util.List;
import java.util.SortedSet;
import org.apache.commons.lang.Validate;
import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.common.homelibrary.home.Home;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.HomeManager;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.jcr.repository.JCRRepository;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem;
import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.client.dsl.ListResolverTyped;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.nodes.Accounting;
import org.gcube.common.storagehub.model.items.nodes.accounting.AccountEntry;
import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.dataharvest.utils.DateUtils;
@ -61,7 +60,7 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
logger.debug("The context is {}", defaultContext);
try {
/*
String vreName = getVRENameToHL(defaultContext);
logger.debug("Getting VRE Name to HL from context/scope returns {} ", vreName);
@ -77,9 +76,15 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
String path = "/Workspace/MySpecialFolders/" + vreName;
logger.debug("Getting item by Path {}", path);
JCRWorkspaceItem item = (JCRWorkspaceItem) ws.getItemByPath(path);
*/
StorageHubClient storageHubClient = new StorageHubClient();
FolderContainer vreFolderContainer = storageHubClient.openVREFolder();
FolderItem vreFolderItem = vreFolderContainer.get();
logger.debug("Analyzing {} in the period [{} to {}] starting from root {}", defaultContext,
DateUtils.format(start), DateUtils.format(end), item.getName());
DateUtils.format(start), DateUtils.format(end), vreFolderItem.getName());
ScopeDescriptor defaultScopeDescriptor = AccountingDataHarvesterPlugin.getScopeDescriptor();
@ -90,17 +95,22 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
ArrayList<AccountingRecord> accountingRecords = new ArrayList<AccountingRecord>();
for(WorkspaceItem children : item.getChildren()) {
ListResolverTyped listResolverTyped = vreFolderContainer.list();
List<ItemContainer<? extends Item>> containers = listResolverTyped.includeHidden().getContainers();
for(ItemContainer<? extends Item> itemContainer : containers) {
count = 0; //resettings the counter
//HarvestedData harvestedData;
//Getting statistics for folder
if(children.isFolder()) {
logger.debug("Getting statistics for folder {}", children.getName());
getStats(children, start, end);
if(itemContainer.getType() == ContainerType.FOLDER) {
Item item = itemContainer.get();
logger.debug("Getting statistics for folder {}", item.getName());
getStats(itemContainer, start, end);
String normalizedName = children.getName().replaceAll("[^A-Za-z0-9]", "");
String normalizedName = item.getName().replaceAll("[^A-Za-z0-9]", "");
String context = mapWsFolderNameToVRE.get(normalizedName);
//Checking if it is a VRE name to right accounting...
if(context != null && !context.isEmpty()) {
@ -147,19 +157,21 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
* @return the stats
* @throws InternalErrorException the internal error exception
*/
private void getStats(WorkspaceItem baseItem, Date start, Date end) throws InternalErrorException {
List<? extends WorkspaceItem> children;
if(baseItem.isFolder()) {
children = baseItem.getChildren();
for(WorkspaceItem child : children)
getStats(child, start, end);
private void getStats(ItemContainer<? extends Item> itemContainer, Date start, Date end) throws Exception {
if(itemContainer.getType() == ContainerType.FOLDER) {
ListResolverTyped listResolverTyped = ((FolderContainer)itemContainer).list();
List<ItemContainer<? extends Item>> containers = listResolverTyped.includeHidden().getContainers();
for(ItemContainer<? extends Item> itemCont : containers) {
getStats(itemCont , start, end);
}
} else {
try {
List<AccountingEntry> accounting = baseItem.getAccounting();
for(AccountingEntry entry : accounting) {
Accounting accounting = itemContainer.get().getAccounting();
for(AccountEntry entry : accounting.getEntries()) {
switch(entry.getEntryType()) {
switch(entry.getType()) {
case CREATE:
case UPDATE:
case READ:
@ -177,26 +189,9 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
}
} catch(Exception e) {
throw new InternalErrorException(e);
throw e;
}
}
}
/**
* Gets the VRE name to HL.
*
* @param vre the vre
* @return the VRE name to HL
*/
private static String getVRENameToHL(String vre) {
Validate.notNull(vre, "scope must be not null");
String newName;
if(vre.startsWith(JCRRepository.PATH_SEPARATOR))
newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-").substring(1);
else
newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-");
return newName;
}
}

7
src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java
View File

@ -1,5 +1,6 @@
package org.gcube.dataharvest.harvester.sobigdata;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@ -15,7 +16,6 @@ import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
public class ResourceCatalogueHarvester extends SoBigDataHarvester {
private static final String AND = " AND ";
private static final String UTF_8_CHARASET = "UTF-8";
public static int ROWS = 500;
@ -207,8 +208,8 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester {
q += solrParameters.get(solrParameters.size() - 1);
}
query += "q=" + UrlEncoderUtil.encodeQuery(q) + "&wt=json&indent=true&rows=" + ROWS;
query += flValue != null && !flValue.isEmpty() ? "&fl=" + UrlEncoderUtil.encodeQuery(flValue) : "";
query += "q=" + URLEncoder.encode(q, UTF_8_CHARASET) + "&wt=json&indent=true&rows=" + ROWS;
query += flValue != null && !flValue.isEmpty() ? "&fl=" + URLEncoder.encode(flValue, UTF_8_CHARASET) : "";
logger.debug("\nPerforming query {}", query);
String jsonResult = Utils.getJson(query);
logger.trace("Response is {}", jsonResult);