diff --git a/pom.xml b/pom.xml index c4a7562..3369141 100644 --- a/pom.xml +++ b/pom.xml @@ -95,15 +95,11 @@ google-http-client-gson 1.21.0 + org.gcube.common - home-library - [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) - - - org.gcube.common - home-library-jcr - [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) + storagehub-client-library + [1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT) @@ -113,7 +109,7 @@ compile - + com.fasterxml.jackson.core jackson-core @@ -154,7 +150,7 @@ common-scope-maps provided - + org.gcube.resources.discovery diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java index 38df26a..a65cf31 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java @@ -7,18 +7,17 @@ import java.util.Date; import java.util.List; import java.util.SortedSet; -import org.apache.commons.lang.Validate; import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; -import org.gcube.common.homelibrary.home.Home; -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.HomeManager; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; -import org.gcube.common.homelibrary.jcr.repository.JCRRepository; -import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace; -import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem; +import org.gcube.common.storagehub.client.dsl.ContainerType; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.client.dsl.ListResolverTyped; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; +import org.gcube.common.storagehub.model.items.FolderItem; +import org.gcube.common.storagehub.model.items.Item; +import org.gcube.common.storagehub.model.items.nodes.Accounting; +import org.gcube.common.storagehub.model.items.nodes.accounting.AccountEntry; import org.gcube.dataharvest.AccountingDataHarvesterPlugin; import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.utils.DateUtils; @@ -61,7 +60,7 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { logger.debug("The context is {}", defaultContext); try { - + /* String vreName = getVRENameToHL(defaultContext); logger.debug("Getting VRE Name to HL from context/scope returns {} ", vreName); @@ -77,9 +76,15 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { String path = "/Workspace/MySpecialFolders/" + vreName; logger.debug("Getting item by Path {}", path); JCRWorkspaceItem item = (JCRWorkspaceItem) ws.getItemByPath(path); + */ + + StorageHubClient storageHubClient = new StorageHubClient(); + FolderContainer vreFolderContainer = storageHubClient.openVREFolder(); + + FolderItem vreFolderItem = vreFolderContainer.get(); logger.debug("Analyzing {} in the period [{} to {}] starting from root {}", defaultContext, - DateUtils.format(start), DateUtils.format(end), item.getName()); + DateUtils.format(start), DateUtils.format(end), vreFolderItem.getName()); ScopeDescriptor defaultScopeDescriptor = AccountingDataHarvesterPlugin.getScopeDescriptor(); @@ -90,17 +95,22 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { ArrayList accountingRecords = new ArrayList(); - for(WorkspaceItem children : item.getChildren()) { + ListResolverTyped listResolverTyped = vreFolderContainer.list(); + List> containers = listResolverTyped.includeHidden().getContainers(); + for(ItemContainer itemContainer : containers) { + + count = 0; //resettings the counter //HarvestedData harvestedData; //Getting statistics for folder - if(children.isFolder()) { - logger.debug("Getting statistics for folder {}", children.getName()); - getStats(children, start, end); + if(itemContainer.getType() == ContainerType.FOLDER) { + Item item = itemContainer.get(); + logger.debug("Getting statistics for folder {}", item.getName()); + getStats(itemContainer, start, end); - String normalizedName = children.getName().replaceAll("[^A-Za-z0-9]", ""); + String normalizedName = item.getName().replaceAll("[^A-Za-z0-9]", ""); String context = mapWsFolderNameToVRE.get(normalizedName); //Checking if it is a VRE name to right accounting... if(context != null && !context.isEmpty()) { @@ -147,19 +157,21 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { * @return the stats * @throws InternalErrorException the internal error exception */ - private void getStats(WorkspaceItem baseItem, Date start, Date end) throws InternalErrorException { - List children; - if(baseItem.isFolder()) { - children = baseItem.getChildren(); - for(WorkspaceItem child : children) - getStats(child, start, end); + private void getStats(ItemContainer itemContainer, Date start, Date end) throws Exception { + if(itemContainer.getType() == ContainerType.FOLDER) { + + ListResolverTyped listResolverTyped = ((FolderContainer)itemContainer).list(); + List> containers = listResolverTyped.includeHidden().getContainers(); + for(ItemContainer itemCont : containers) { + getStats(itemCont , start, end); + } + } else { try { - - List accounting = baseItem.getAccounting(); - for(AccountingEntry entry : accounting) { + Accounting accounting = itemContainer.get().getAccounting(); + for(AccountEntry entry : accounting.getEntries()) { - switch(entry.getEntryType()) { + switch(entry.getType()) { case CREATE: case UPDATE: case READ: @@ -177,26 +189,9 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { } } catch(Exception e) { - throw new InternalErrorException(e); + throw e; } } } - /** - * Gets the VRE name to HL. - * - * @param vre the vre - * @return the VRE name to HL - */ - private static String getVRENameToHL(String vre) { - Validate.notNull(vre, "scope must be not null"); - - String newName; - if(vre.startsWith(JCRRepository.PATH_SEPARATOR)) - newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-").substring(1); - else - newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-"); - return newName; - } - } diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java index 76c0dd2..1911dd5 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java @@ -1,5 +1,6 @@ package org.gcube.dataharvest.harvester.sobigdata; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -15,7 +16,6 @@ import org.gcube.dataharvest.AccountingDataHarvesterPlugin; import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.Utils; -import org.gcube.portlets.user.urlshortener.UrlEncoderUtil; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory; public class ResourceCatalogueHarvester extends SoBigDataHarvester { private static final String AND = " AND "; + private static final String UTF_8_CHARASET = "UTF-8"; public static int ROWS = 500; @@ -207,8 +208,8 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester { q += solrParameters.get(solrParameters.size() - 1); } - query += "q=" + UrlEncoderUtil.encodeQuery(q) + "&wt=json&indent=true&rows=" + ROWS; - query += flValue != null && !flValue.isEmpty() ? "&fl=" + UrlEncoderUtil.encodeQuery(flValue) : ""; + query += "q=" + URLEncoder.encode(q, UTF_8_CHARASET) + "&wt=json&indent=true&rows=" + ROWS; + query += flValue != null && !flValue.isEmpty() ? "&fl=" + URLEncoder.encode(flValue, UTF_8_CHARASET) : ""; logger.debug("\nPerforming query {}", query); String jsonResult = Utils.getJson(query); logger.trace("Response is {}", jsonResult);