gCubeSystem
/
accounting-dashboard-harvester-se-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Migrating from HomeLibrary to StorageHub refs #17128

This commit is contained in:
Luca Frosini 2019-07-15 17:43:18 +02:00
parent 1a8c8f864a
commit 2977a684fd
3 changed files with 49 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pom.xml
View File

@ -95,15 +95,11 @@
<artifactId>google-http-client-gson</artifactId> <artifactId>google-http-client-gson</artifactId>
<version>1.21.0</version> <version>1.21.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.gcube.common</groupId> <groupId>org.gcube.common</groupId>
<artifactId>home-library</artifactId> <artifactId>storagehub-client-library</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version> <version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library-jcr</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency> </dependency>
<dependency> <dependency>
@ -113,7 +109,7 @@
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<!-- Dependencies force to provided --> <!-- Dependencies forced to provided -->
<dependency> <dependency>
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId> <artifactId>jackson-core</artifactId>
@ -154,7 +150,7 @@
<artifactId>common-scope-maps</artifactId> <artifactId>common-scope-maps</artifactId>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<!-- END Dependencies force to provided --> <!-- END Dependencies forced to provided -->
<dependency> <dependency>
<groupId>org.gcube.resources.discovery</groupId> <groupId>org.gcube.resources.discovery</groupId>

85
src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java
View File

@ -7,18 +7,17 @@ import java.util.Date;
import java.util.List; import java.util.List;
import java.util.SortedSet; import java.util.SortedSet;
import org.apache.commons.lang.Validate;
import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.common.homelibrary.home.Home; import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.homelibrary.home.HomeLibrary; import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.homelibrary.home.HomeManager; import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.storagehub.client.dsl.ListResolverTyped;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.homelibrary.jcr.repository.JCRRepository; import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace; import org.gcube.common.storagehub.model.items.nodes.Accounting;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem; import org.gcube.common.storagehub.model.items.nodes.accounting.AccountEntry;
import org.gcube.dataharvest.AccountingDataHarvesterPlugin; import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.DateUtils;
@ -61,7 +60,7 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
logger.debug("The context is {}", defaultContext); logger.debug("The context is {}", defaultContext);
try { try {
/*
String vreName = getVRENameToHL(defaultContext); String vreName = getVRENameToHL(defaultContext);
logger.debug("Getting VRE Name to HL from context/scope returns {} ", vreName); logger.debug("Getting VRE Name to HL from context/scope returns {} ", vreName);
@ -77,9 +76,15 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
String path = "/Workspace/MySpecialFolders/" + vreName; String path = "/Workspace/MySpecialFolders/" + vreName;
logger.debug("Getting item by Path {}", path); logger.debug("Getting item by Path {}", path);
JCRWorkspaceItem item = (JCRWorkspaceItem) ws.getItemByPath(path); JCRWorkspaceItem item = (JCRWorkspaceItem) ws.getItemByPath(path);
*/
StorageHubClient storageHubClient = new StorageHubClient();
FolderContainer vreFolderContainer = storageHubClient.openVREFolder();
FolderItem vreFolderItem = vreFolderContainer.get();
logger.debug("Analyzing {} in the period [{} to {}] starting from root {}", defaultContext, logger.debug("Analyzing {} in the period [{} to {}] starting from root {}", defaultContext,
DateUtils.format(start), DateUtils.format(end), item.getName()); DateUtils.format(start), DateUtils.format(end), vreFolderItem.getName());
ScopeDescriptor defaultScopeDescriptor = AccountingDataHarvesterPlugin.getScopeDescriptor(); ScopeDescriptor defaultScopeDescriptor = AccountingDataHarvesterPlugin.getScopeDescriptor();
@ -90,17 +95,22 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
ArrayList<AccountingRecord> accountingRecords = new ArrayList<AccountingRecord>(); ArrayList<AccountingRecord> accountingRecords = new ArrayList<AccountingRecord>();
for(WorkspaceItem children : item.getChildren()) { ListResolverTyped listResolverTyped = vreFolderContainer.list();
List<ItemContainer<? extends Item>> containers = listResolverTyped.includeHidden().getContainers();
for(ItemContainer<? extends Item> itemContainer : containers) {
count = 0; //resettings the counter count = 0; //resettings the counter
//HarvestedData harvestedData; //HarvestedData harvestedData;
//Getting statistics for folder //Getting statistics for folder
if(children.isFolder()) { if(itemContainer.getType() == ContainerType.FOLDER) {
logger.debug("Getting statistics for folder {}", children.getName()); Item item = itemContainer.get();
getStats(children, start, end); logger.debug("Getting statistics for folder {}", item.getName());
getStats(itemContainer, start, end);
String normalizedName = children.getName().replaceAll("[^A-Za-z0-9]", ""); String normalizedName = item.getName().replaceAll("[^A-Za-z0-9]", "");
String context = mapWsFolderNameToVRE.get(normalizedName); String context = mapWsFolderNameToVRE.get(normalizedName);
//Checking if it is a VRE name to right accounting... //Checking if it is a VRE name to right accounting...
if(context != null && !context.isEmpty()) { if(context != null && !context.isEmpty()) {
@ -147,19 +157,21 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
* @return the stats * @return the stats
* @throws InternalErrorException the internal error exception * @throws InternalErrorException the internal error exception
*/ */
private void getStats(WorkspaceItem baseItem, Date start, Date end) throws InternalErrorException { private void getStats(ItemContainer<? extends Item> itemContainer, Date start, Date end) throws Exception {
List<? extends WorkspaceItem> children; if(itemContainer.getType() == ContainerType.FOLDER) {
if(baseItem.isFolder()) {
children = baseItem.getChildren(); ListResolverTyped listResolverTyped = ((FolderContainer)itemContainer).list();
for(WorkspaceItem child : children) List<ItemContainer<? extends Item>> containers = listResolverTyped.includeHidden().getContainers();
getStats(child, start, end); for(ItemContainer<? extends Item> itemCont : containers) {
getStats(itemCont , start, end);
}
} else { } else {
try { try {
Accounting accounting = itemContainer.get().getAccounting();
for(AccountEntry entry : accounting.getEntries()) {
List<AccountingEntry> accounting = baseItem.getAccounting(); switch(entry.getType()) {
for(AccountingEntry entry : accounting) {
switch(entry.getEntryType()) {
case CREATE: case CREATE:
case UPDATE: case UPDATE:
case READ: case READ:
@ -177,26 +189,9 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
} }
} catch(Exception e) { } catch(Exception e) {
throw new InternalErrorException(e); throw e;
} }
} }
} }
/**
* Gets the VRE name to HL.
*
* @param vre the vre
* @return the VRE name to HL
*/
private static String getVRENameToHL(String vre) {
Validate.notNull(vre, "scope must be not null");
String newName;
if(vre.startsWith(JCRRepository.PATH_SEPARATOR))
newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-").substring(1);
else
newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-");
return newName;
}
} }

7
src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java
View File

@ -1,5 +1,6 @@
package org.gcube.dataharvest.harvester.sobigdata; package org.gcube.dataharvest.harvester.sobigdata;
import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
@ -15,7 +16,6 @@ import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.Utils; import org.gcube.dataharvest.utils.Utils;
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
public class ResourceCatalogueHarvester extends SoBigDataHarvester { public class ResourceCatalogueHarvester extends SoBigDataHarvester {
private static final String AND = " AND "; private static final String AND = " AND ";
private static final String UTF_8_CHARASET = "UTF-8";
public static int ROWS = 500; public static int ROWS = 500;
@ -207,8 +208,8 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester {
q += solrParameters.get(solrParameters.size() - 1); q += solrParameters.get(solrParameters.size() - 1);
} }
query += "q=" + UrlEncoderUtil.encodeQuery(q) + "&wt=json&indent=true&rows=" + ROWS; query += "q=" + URLEncoder.encode(q, UTF_8_CHARASET) + "&wt=json&indent=true&rows=" + ROWS;
query += flValue != null && !flValue.isEmpty() ? "&fl=" + UrlEncoderUtil.encodeQuery(flValue) : ""; query += flValue != null && !flValue.isEmpty() ? "&fl=" + URLEncoder.encode(flValue, UTF_8_CHARASET) : "";
logger.debug("\nPerforming query {}", query); logger.debug("\nPerforming query {}", query);
String jsonResult = Utils.getJson(query); String jsonResult = Utils.getJson(query);
logger.trace("Response is {}", jsonResult); logger.trace("Response is {}", jsonResult);