gCubeSystem
/
uri-resolver
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Bug fixing LoadingMapOfDetachedVRE

This commit is contained in:
Francesco Mangiacrapa 2020-06-10 14:46:02 +02:00
parent c4ff6be857
commit d96186d0e9
6 changed files with 81 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
src/main/java/org/gcube/datatransfer/resolver/caches/LoadingMapOfDetachedVRE.java
View File

@ -3,7 +3,7 @@
*/
package org.gcube.datatransfer.resolver.caches;
import java.util.concurrent.ConcurrentMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@ -23,15 +23,16 @@ import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
/**
* The Class LoadingMapOfDetachedVRE.
*
* @author Francesco Mangiacrapa at ISTI-CNR Pisa (Italy)
* Jun 8, 2020
* Jun 10, 2020
*/
public class LoadingMapOfDetachedVRE {
private static Logger logger = LoggerFactory.getLogger(LoadingMapOfDetachedVRE.class);
private static Logger LOG = LoggerFactory.getLogger(LoadingMapOfDetachedVRE.class);
private static LoadingCache<String, VRE> vreNameToVRE;
static{
@ -42,9 +43,14 @@ public class LoadingMapOfDetachedVRE {
public VRE load(String vreName)
throws Exception {
logger.info("Loading the cache for VRE name: {}",vreName);
VRE theVRE = loadFullScopeforScopeName(vreName);
logger.info("Returning {} with scope {} for the VRE name: {}", VRE.class.getSimpleName(), theVRE.getScope(), vreName);
LOG.info("Loading the cache for VRE name: {}",vreName);
VRE theVRE = loadVREObjForVREName(vreName);
if(theVRE!=null)
LOG.info("Returning {} with scope {} for the VRE name: {}", VRE.class.getSimpleName(), theVRE.getScope(), vreName);
else {
LOG.info("No VRE obj for VRE name {}",vreName);
}
return theVRE;
}
@ -55,20 +61,18 @@ public class LoadingMapOfDetachedVRE {
@Override
public void onRemoval(RemovalNotification<String, VRE> arg0) {
logger.info("cache expired");
//prePopulateCache();
LOG.info("cache expired");
}
};
vreNameToVRE = CacheBuilder.newBuilder().maximumSize(300).expireAfterWrite(
1, TimeUnit.DAYS).removalListener(removalListener).
12, TimeUnit.HOURS).removalListener(removalListener).
build(loader);
//Populating the cache at init stage
populateTheCache();
logger.info("Pre-Loaded detached VRE Name to VRE Obj cache with: "+vreNameToVRE.asMap().size()+" item/s");
LOG.info("Pre-Loaded detached VRE Name to VRE Obj cache with: "+vreNameToVRE.asMap().size()+" item/s");
}
@ -76,22 +80,23 @@ public class LoadingMapOfDetachedVRE {
* Populate the cache.
*/
private static void populateTheCache(){
try{
//POPULATE THE CACHE READING THE RESOURCE "CATALOGUE-RESOLVER"
logger.info("Trying to pre-populate the detached VREs cache with mapping (VRE Name, VRE Obj)");
//Populating the cache by using the detachedres-library
LOG.info("Trying to pre-populate the detached VREs cache with mapping (VRE Name, VRE Obj)");
ScopeProvider.instance.set(UriResolverSmartGearManagerInit.getRootContextScope());
DetachedREsClient detachedREsClient = new DetachedREsClient();
DetachedREs detachedREs = detachedREsClient.getDetachedREs();
int totalVREDimissed = 0;
for (Gateway gateway : detachedREs.getGateways().values()) {
logger.trace("Gateway: " + gateway.getName());
LOG.trace("Gateway: " + gateway.getName());
int vreDismissedPerGatew = 0;
for (VO vo : gateway.getVos().values()) {
logger.trace("VO: " + vo.getName());
LOG.trace("VO: " + vo.getName());
for (VRE vre : vo.getVres().values()) {
if(logger.isTraceEnabled()) {
logger.trace("VRE name: " + vre.getName() +
if(LOG.isTraceEnabled()) {
LOG.trace("VRE name: " + vre.getName() +
" scope: "+vre.getScope() +
" VRE catalogue url: " + vre.getCatalogUrl() +
" VRE catalog Portlet URL: "+vre.getCatalogPortletURL());
@ -99,28 +104,30 @@ public class LoadingMapOfDetachedVRE {
try {
if(vre.getScope()!=null && !vre.getScope().isEmpty()) {
String vreName = vre.getScope().split("/")[3];
vreNameToVRE.put(vreName, vre);
vreNameToVRE.asMap().put(vreName, vre);
vreDismissedPerGatew++;
}
}catch (Exception e) {
logger.warn("Error on parsing the scope: "+vre.getScope()+ " skipping it");
LOG.warn("Error on parsing the scope: "+vre.getScope()+ " skipping it");
}
}
}
logger.debug("\nVREs dismissed loaded and cached per " + gateway.getName() + " are: "+vreDismissedPerGatew);
LOG.debug("VREs dismissed loaded and cached per " + gateway.getName() + " are: "+vreDismissedPerGatew);
totalVREDimissed+=vreDismissedPerGatew;
}
logger.debug("\nTotal detached VREs are: "+totalVREDimissed);
LOG.debug("Total detached VREs are: "+totalVREDimissed);
logger.info("Cache populated with: ");
ConcurrentMap<String, VRE> mapOfVreNames = vreNameToVRE.asMap();
LOG.info("Cache populated with: ");
Map<String, VRE> mapOfVreNames = vreNameToVRE.asMap();
for (String key : mapOfVreNames.keySet()) {
VRE theDetachedVRE = mapOfVreNames.get(key);
logger.info("detached VRE with scope {}, catalogueURL {}, cataloguePortletURL {}",
theDetachedVRE.getScope(), theDetachedVRE.getCatalogUrl(),
LOG.info("VRE name {}, VRE obj with scope {}, catalogueURL {}, cataloguePortletURL {}",
key,
theDetachedVRE.getScope(),
theDetachedVRE.getCatalogUrl(),
theDetachedVRE.getCatalogPortletURL());
}
@ -132,8 +139,9 @@ public class LoadingMapOfDetachedVRE {
}
/**
* Gets the.
* Gets the VRE obj for input VRE name.
*
* @param vreName the vre name
* @return the vre
@ -141,35 +149,25 @@ public class LoadingMapOfDetachedVRE {
*/
public static VRE get(String vreName) throws ExecutionException{
boolean cacheNotLoaded = true;
final int MAX_ATTEMPTS = 3;
int retry = 1;
while (cacheNotLoaded) {
cacheNotLoaded = vreNameToVRE == null;
if(retry<MAX_ATTEMPTS) {
try {
Thread.sleep(retry * 1000);
} catch (InterruptedException e) {
}
}
retry++;
logger.debug("is cache loaded? {}",!cacheNotLoaded);
try {
return vreNameToVRE.get(vreName);
}catch (Exception e) {
LOG.info("Error on getting VRE obj for vreName {}. Is the key {} not found in the cache?", vreName, vreName);
throw e;
}
return vreNameToVRE.get(vreName);
}
/**
* Load full scopefor scope name.
* Load VRE obj for VRE name.
*
* @param vreName the scope name
* @return the scope bean
* @param vreName the vre name
* @return the vre
*/
protected static VRE loadFullScopeforScopeName(String vreName){
protected static VRE loadVREObjForVREName(String vreName){
//THIS CHECK SHOULD BE NOT NEEDED
VRE theVRE = vreNameToVRE.getIfPresent(vreName);
//THIS CHECK SHOULD NOT BE NEEDED
if(theVRE==null){
populateTheCache();
theVRE = vreNameToVRE.getIfPresent(vreName);

9
src/main/java/org/gcube/datatransfer/resolver/caches/LoadingMapOfScopeCache.java
View File

@ -86,13 +86,6 @@ public class LoadingMapOfScopeCache {
}
logger.info("Cache populated with: "+scopeNamesToFullScopes.asMap().toString());
//logger.info("Pre-Loaded CatalogueApplicationProfiles cache is: "+catalogueApplicationProfiles.asMap().toString());
// if(UriResolverSmartGearManagerInit.getRootContextScope().compareTo("/gcube")==0){
// logger.warn("HARD-CABLING PARTHENOS_Registry scope to resolve PARTHENOS_REGISTRY Links in dev environment");
// scopeNamesToFullScopes.asMap().put("PARTHENOS_Registry", "/d4science.research-infrastructures.eu/ParthenosVO/PARTHENOS_Registry");
// }
}catch(Exception e){
//SILENT
@ -123,9 +116,9 @@ public class LoadingMapOfScopeCache {
*/
protected static ScopeBean loadFullScopeforScopeName(String scopeName){
//THIS CHECK SHOULD BE NOT NEEDED
ScopeBean fullScope = scopeNamesToFullScopes.getIfPresent(scopeName);
//THIS CHECK SHOULD NOT BE NEEDED
if(fullScope==null){
populateTheCache();
fullScope = scopeNamesToFullScopes.getIfPresent(scopeName);

2
src/main/java/org/gcube/datatransfer/resolver/init/UriResolverSmartGearManagerInit.java
View File

@ -17,6 +17,7 @@ import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.datatransfer.resolver.caches.LoadingGeonetworkInstanceCache;
import org.gcube.datatransfer.resolver.caches.LoadingGisViewerApplicationURLCache;
import org.gcube.datatransfer.resolver.caches.LoadingMapOfDetachedVRE;
import org.gcube.datatransfer.resolver.caches.LoadingMapOfScopeCache;
import org.gcube.datatransfer.resolver.gis.property.ApplicationProfilePropertyReader;
import org.gcube.datatransfer.resolver.gis.property.PropertyFileNotFoundException;
@ -94,6 +95,7 @@ public class UriResolverSmartGearManagerInit implements ApplicationManager {
new LoadingGeonetworkInstanceCache();
new LoadingGisViewerApplicationURLCache();
new LoadingMapOfScopeCache();
new LoadingMapOfDetachedVRE();
}
}
catch (Exception e) {

25
src/test/java/DetachedVREs.java → src/test/java/DetachedVREsTest.java
View File

@ -1,31 +1,44 @@
import java.util.concurrent.ExecutionException;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datatransfer.resolver.caches.LoadingMapOfDetachedVRE;
import org.gcube.infrastructure.detachedres.detachedreslibrary.shared.re.VRE;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DetachedVREs {
public class DetachedVREsTest {
public static Logger LOG = LoggerFactory.getLogger(DetachedVREs.class);
public static Logger LOG = LoggerFactory.getLogger(DetachedVREsTest.class);
public static String TOKEN = "0e2c7963-8d3e-4ea6-a56d-ffda530dd0fa-98187548";
public static String TOKEN = "";
public static String detachedVREName = "BlueBridgeProject";
public static String rootScope = "/gcube";
//@Before
public void initCache() {
}
/**
* Storage hub test.
*
* @throws Exception the exception
*/
@Test
//@Test
public void testCacheOfDetachedVREs() throws Exception{
try {
ScopeProvider.instance.set(rootScope);
SecurityTokenProvider.instance.set(TOKEN);
VRE theDetachedVRE = LoadingMapOfDetachedVRE.get("NextNext");
//LoadingMapOfDetachedVRE cache = new LoadingMapOfDetachedVRE();
VRE theDetachedVRE = LoadingMapOfDetachedVRE.get(detachedVREName);
LOG.info("Detached VRE found {}", theDetachedVRE);
}catch (ExecutionException e) {
LOG.info("The Detached VRE for name {} not found",detachedVREName);
}catch (Exception e) {
e.printStackTrace();
}
}

28
src/test/java/GetAllInfrastructureScopesFromIS.java
View File

@ -3,7 +3,6 @@ import static org.gcube.resources.discovery.icclient.ICFactory.client;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@ -18,6 +17,7 @@ import org.gcube.datatransfer.resolver.init.UriResolverSmartGearManagerInit;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.Query;
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
@ -38,12 +38,12 @@ public class GetAllInfrastructureScopesFromIS {
protected static final String RESOURCE_PROFILE_NAME_TEXT = "/Resource/Profile/Name/text()";
/**
* The main method.
*
* @param args the arguments
* Test loading map of scope.
*/
public static void main(String[] args) {
@Test
public void testLoadingMapOfScope() {
try {
@ -53,16 +53,16 @@ public class GetAllInfrastructureScopesFromIS {
UriResolverSmartGearManagerInit.setRootContextScope(rootScope);
LoadingMapOfScopeCache cache = new LoadingMapOfScopeCache();
int i = 0;
for (String string : cache.getCache().asMap().keySet()) {
try {
System.out.println(++i+") Scope Name: "+string + " to full scope: "+LoadingMapOfScopeCache.get(string));
} catch (ExecutionException e) {
e.printStackTrace();
}
}
// int i = 0;
// for (String string : cache.getCache().asMap().keySet()) {
// try {
// System.out.println(++i+") Scope Name: "+string + " to full scope: "+LoadingMapOfScopeCache.get(string));
// } catch (ExecutionException e) {
// e.printStackTrace();
// }
// }
String scopeName = "devsec";
String scopeName = "abc";
ScopeBean scopeBean = LoadingMapOfScopeCache.get(scopeName);
String fullScope = scopeBean.toString();
logger.info("Read fullScope: "+fullScope + " for SCOPE name: "+scopeName +" from cache created by: "+GetAllInfrastructureScopes.class.getSimpleName());

2
src/test/java/GisResolverTest.java
View File

@ -13,7 +13,7 @@ import org.gcube.datatransfer.resolver.services.GisResolver;
public class GisResolverTest {
static String scope = "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab";
static String gisUUID = "";
static String gisUUID = "6b99efdf-2202-4b6f-aaa3-7e10e0bf09f4";
public static void main(String[] args) {
GisResolver gisResolver = new GisResolver();
ScopeProvider.instance.set(scope);