package org.gcube.datacatalogue.grsf_manage_widget.server.manage;

import static org.gcube.resources.discovery.icclient.ICFactory.client;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;

import java.io.StringReader;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpSession;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import org.gcube.common.encryption.StringEncrypter;
import org.gcube.common.portal.PortalContext;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.resources.gcore.utils.XPathHelper;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datacatalogue.ckanutillibrary.server.ApplicationProfileScopePerUrlReader;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueRunningCluster;
import org.gcube.datacatalogue.ckanutillibrary.shared.ex.ApplicationProfileNotFoundException;
import org.gcube.datacatalogue.common.Constants;
import org.gcube.datacatalogue.common.enums.Status;
import org.gcube.datacatalogue.grsf_manage_widget.shared.ConnectedBean;
import org.gcube.datacatalogue.grsf_manage_widget.shared.ManageProductBean;
import org.gcube.datacatalogue.grsf_manage_widget.shared.RevertOperationUrl;
import org.gcube.datacatalogue.grsf_manage_widget.shared.RevertOperationUrl.Operation;
import org.gcube.datacatalogue.grsf_manage_widget.shared.SimilarGRSFRecord;
import org.gcube.portlets.user.urlshortener.UrlShortener;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.Query;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
import org.gcube.vomanagement.usermanagement.RoleManager;
import org.gcube.vomanagement.usermanagement.UserManager;
import org.gcube.vomanagement.usermanagement.impl.LiferayRoleManager;
import org.gcube.vomanagement.usermanagement.impl.LiferayUserManager;
import org.gcube.vomanagement.usermanagement.model.GCubeTeam;
import org.gcube.vomanagement.usermanagement.model.GCubeUser;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;

import com.liferay.portal.service.UserLocalServiceUtil;

import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpResponse;
import eu.trentorise.opendata.jackan.internal.org.apache.http.client.methods.HttpPost;
import eu.trentorise.opendata.jackan.internal.org.apache.http.entity.StringEntity;
import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.CloseableHttpClient;
import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.HttpClientBuilder;
import eu.trentorise.opendata.jackan.internal.org.apache.http.util.EntityUtils;
import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanPair;

/**
 * Utility methods for GRSF Management panel widget.
 * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
 */
public class Utils {

	private static final String GENERIC_RESOURCE_NAME = "GRSFManageEntries";
	private static final String GENERIC_RESOURCE_SECONDARY_TYPE = "ApplicationProfile";
	private static final Logger logger = LoggerFactory.getLogger(Utils.class);
	private static final String REGEX_UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";

	/**
	 * Look up from the IS other information that can be potentially displayed in read only mode in the management panel.
	 * @return a list of extra keys to show.
	 */
	public static Set<String> getLookedUpExtrasKeys() {
		Set<String> lookedUpExtrasKeys = new HashSet<String>();
		String scope = ScopeProvider.instance.get();
		logger.debug("Trying to fetch applicationProfile profile from the infrastructure for " + GENERIC_RESOURCE_NAME + " scope: " +  scope);

		try {
			Query q = new QueryBox("for $profile in collection('/db/Profiles/GenericResource')//Resource " +
					"where $profile/Profile/SecondaryType/string() eq '"+ GENERIC_RESOURCE_SECONDARY_TYPE + "' and  $profile/Profile/Name/string() " +
					" eq '" + GENERIC_RESOURCE_NAME + "'" +
					"return $profile");

			DiscoveryClient<String> client = client();
			List<String> appProfile = client.submit(q);

			if (appProfile == null || appProfile.size() == 0) 
				throw new ApplicationProfileNotFoundException("Your applicationProfile is not registered in the infrastructure");
			else {

				String elem = appProfile.get(0);
				DocumentBuilder docBuilder =  DocumentBuilderFactory.newInstance().newDocumentBuilder();
				Node node = docBuilder.parse(new InputSource(new StringReader(elem))).getDocumentElement();
				XPathHelper helper = new XPathHelper(node);

				List<String> currValue = null;
				currValue = helper.evaluate("/Resource/Profile/Body/text()");

				if (currValue != null && currValue.size() > 0) {
					String body = currValue.get(0);
					String[] splittedSet = body.split(",");
					if(splittedSet != null && splittedSet.length > 0)
						for (String entry : splittedSet) {
							String trimmed = entry.trim();
							if(trimmed.isEmpty())
								continue;
							lookedUpExtrasKeys.add(trimmed);
						}
				} 
			}

			logger.info("Extras entries are " + lookedUpExtrasKeys);
			return lookedUpExtrasKeys;
		} catch (Exception e) {
			logger.error("Error while trying to fetch applicationProfile profile from the infrastructure", e);
			return null;
		} 
	}

	/**
	 * Return a map for converting a key to a namespace:key format by reading a generic resource.
	 * @param httpSession 
	 * @return a map
	 */
	@SuppressWarnings("unchecked")
	public static Map<String, String> getFieldToFieldNameSpaceMapping(HttpSession httpSession, String resourceName){

		// check if this information is available in session
		String sessionKey = ScopeProvider.instance.get() + resourceName;
		if(httpSession.getAttribute(sessionKey) != null)
			return (Map<String, String>) httpSession.getAttribute(sessionKey);

		Map<String, String> namespacesMap = new HashMap<String, String>(); // e.g. fishery_identity:Short Title -> Short Title
		try {
			Query q = new QueryBox("for $profile in collection('/db/Profiles/GenericResource')//Resource " +
					"where $profile/Profile/SecondaryType/string() eq '"+ "ApplicationProfile" + "' and  $profile/Profile/Name/string() " +
					" eq '" + resourceName + "'" +
					"return $profile");

			DiscoveryClient<String> client = client();
			List<String> appProfile = client.submit(q);

			if (appProfile == null || appProfile.size() == 0) 
				throw new Exception("Your applicationProfile is not registered in the infrastructure");
			else {

				String elem = appProfile.get(0);
				DocumentBuilder docBuilder =  DocumentBuilderFactory.newInstance().newDocumentBuilder();
				Node node = docBuilder.parse(new InputSource(new StringReader(elem))).getDocumentElement();
				XPathHelper helper = new XPathHelper(node);

				NodeList nodeListKeys = helper.evaluateForNodes("//originalKey");
				NodeList nodeListModifiedKeys = helper.evaluateForNodes("//modifiedKey");
				int sizeKeys = nodeListKeys != null ? nodeListKeys.getLength() : 0;
				int sizeKeysModifed = nodeListModifiedKeys != null ? nodeListModifiedKeys.getLength() : 0;
				if(sizeKeys != sizeKeysModifed)
					throw new Exception("Malformed XML");
				logger.debug("Size is " + sizeKeys);
				for (int i = 0; i < sizeKeys; i++) {
					namespacesMap.put(nodeListModifiedKeys.item(i).getTextContent(), nodeListKeys.item(i).getTextContent());
				}
			}
			logger.debug("Map is " + namespacesMap);
			httpSession.setAttribute(sessionKey, namespacesMap);
			return namespacesMap;
		} catch (Exception e) {
			logger.error("Error while trying to fetch applicationProfile profile from the infrastructure", e);
			return null;
		}
	}

	/**
	 * Replace the extras' keys if needed, e.g. fishery_identity:Short Title -> Short Title
	 * @param extrasAsPairs
	 * @param namespaces
	 * @return a map with replaced key value pairs
	 */
	public static Map<String, List<String>> replaceFieldsKey(List<CkanPair> extrasAsPairs,
			Map<String, String> namespaces) {

		Map<String, List<String>> toReturn = new HashMap<String, List<String>>();

		for (CkanPair ckanPair : extrasAsPairs) {
			String pairKey = ckanPair.getKey();
			String pairValue = ckanPair.getValue();
			String replacedKey = namespaces.containsKey(pairKey) ? namespaces.get(pairKey) : pairKey;

			List<String> values = null;
			if(toReturn.containsKey(replacedKey))
				values = toReturn.get(replacedKey);
			else
				values = new ArrayList<String>(1);

			values.add(pairValue);

			toReturn.put(replacedKey, values);
		}


		return toReturn;
	}

	/**
	 * Get the extras of this dataset as hashmap
	 * @param extrasAsPairs
	 * @return
	 */
	public static Map<String, List<String>> getExtras(List<CkanPair> extrasAsPairs){

		Map<String, List<String>> toReturn = new HashMap<String, List<String>>();

		for (CkanPair ckanPair : extrasAsPairs) {
			String pairKey = ckanPair.getKey();
			String pairValue = ckanPair.getValue();

			List<String> values = null;
			if(toReturn.containsKey(pairKey))
				values = toReturn.get(pairKey);
			else
				values = new ArrayList<String>(1);

			values.add(pairValue);

			toReturn.put(pairKey, values);
		}
		return toReturn;
	}

	/**
	 * Discover the service endpoint and return its url
	 * @param context
	 * @return the url of the service on success, null otherwise
	 */
	public static String discoverEndPoint(String context){

		String oldContext = ScopeProvider.instance.get();
		ScopeProvider.instance.set(context);
		String toReturn = null;
		try{
			SimpleQuery query = queryFor(ServiceEndpoint.class);
			query.addCondition("$resource/Profile/Name/text() eq '"+ Constants.SERVICE_NAME +"'");
			query.addCondition("$resource/Profile/Category/text() eq '"+ Constants.SERVICE_CATEGORY +"'");
			DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
			List<ServiceEndpoint> resources = client.submit(query);

			if (resources.size() == 0){
				logger.error("There is no Runtime Resource having name " + Constants.SERVICE_NAME +" and Category " + Constants.SERVICE_CATEGORY + " in this scope.");
				throw new Exception("There is no Runtime Resource having name " + Constants.SERVICE_NAME +" and Category " + Constants.SERVICE_CATEGORY + " in this scope.");
			}
			else {

				for (ServiceEndpoint res : resources) {

					Iterator<AccessPoint> accessPointIterator = res.profile().accessPoints().iterator();

					while (accessPointIterator.hasNext()) {
						ServiceEndpoint.AccessPoint accessPoint = (ServiceEndpoint.AccessPoint) accessPointIterator
								.next();

						// return the path
						toReturn = accessPoint.address();
					}
				}
			}
		}catch(Exception e){
			logger.error("Unable to retrieve such service endpoint information!", e);
		}finally{
			if(oldContext != null && !oldContext.equals(context))
				ScopeProvider.instance.set(oldContext);
		}

		return toReturn;
	}

	/**
	 * Send an update for this bean
	 * @param baseUrl
	 * @param bean
	 * @param username 
	 * @param catalogue 
	 * @return true on success, false otherwise
	 */
	public static String updateRecord(String serviceUrl, ManageProductBean bean, DataCatalogue catalogue, String username,
			String fullName, HttpServletRequest httpServletRequest, long groupId) throws Exception{

		if(serviceUrl == null)
			throw new IllegalArgumentException("GRSF Updater service url cannot be null");

		if(bean == null)
			throw new IllegalArgumentException("Item bean to manage cannot be null");

		try(CloseableHttpClient httpClient = HttpClientBuilder.create().build();){

			// if there are merges, update the status of the involved records immediately
			if(bean.isMergesInvolved())
				updateStatusInvolvedRecords(bean, catalogue);

			// send update to the knowledge base
			updateKB(httpClient, serviceUrl, bean, catalogue, username, fullName);

			// send email to Editors and Reviewers if merges are involved or the record was rejected (but the record was the result of a merge) TODO
			if(bean.isMergesInvolved() || bean.getNewStatus().equals(Status.Rejected))
				sendEmailAdministrators(bean, catalogue, username, fullName, groupId, httpServletRequest);

			// create a post about the operation
			createSocialPost(bean, catalogue, username, fullName);

		}catch(Exception e){
			logger.error("Unable to update this Item ", e);
			throw e;
		}
		return null;
	}

	/**
	 * Update the status of the involved records to "to be merged"
	 * @param bean
	 * @param catalogue
	 * @param username
	 * @param fullName
	 * @throws Exception 
	 */
	private static void updateStatusInvolvedRecords(ManageProductBean bean, DataCatalogue catalogue) throws Exception {

		String context = ScopeProvider.instance.get();
		String sysApi = fetchSysAPI(context);
		for(SimilarGRSFRecord s: bean.getSimilarGrsfRecords()){
			if(s.isSuggestedMerge()){
				String productId = s.getKnowledgeBaseId();
				Map<String, List<String>> extrasMap = getExtras(catalogue.getDataset(productId, sysApi).getExtras());
				extrasMap.put(Constants.STATUS_OF_THE_GRSF_RECORD_CUSTOM_KEY, Arrays.asList(Status.To_be_Merged.getOrigName()));
				catalogue.patchProductCustomFields(productId, sysApi, extrasMap);
			}
		}

		// update the current status record
		String productId = bean.getKnowledgeBaseIdentifier();
		Map<String, List<String>> extrasMap = getExtras(catalogue.getDataset(productId, sysApi).getExtras());
		extrasMap.put(Constants.STATUS_OF_THE_GRSF_RECORD_CUSTOM_KEY, Arrays.asList(Status.To_be_Merged.getOrigName()));
		catalogue.patchProductCustomFields(productId, sysApi, extrasMap);

	}

	/**
	 * Create a post with proper hashtags of the action taken by who and on which record
	 * @param bean
	 * @param catalogue
	 * @param username
	 * @param fullName
	 */
	private static void createSocialPost(ManageProductBean bean,
			DataCatalogue catalogue, String username, String fullName) {
		List<String> hashtags = getHashTagsFromActions(bean);
		// TODO

	}

	/**
	 * Send an email to the administrator as well as the 
	 * @param bean
	 * @param catalogue
	 * @param username
	 * @param fullName
	 * @param httpSession 
	 * @throws Exceptio 
	 */
	private static void sendEmailAdministrators(ManageProductBean bean,
			DataCatalogue catalogue, String username, String fullName, long groupId, HttpServletRequest httpServletRequest) throws Exception {

		// get the list of GRSF Reviewers to alert as well
		RoleManager roleManager = new LiferayRoleManager();
		List<GCubeTeam> teamRoles = roleManager.listTeamsByGroup(groupId);
		List<GCubeUser> reviewers = new ArrayList<>();
		UserManager um = new LiferayUserManager();

		for(GCubeTeam tr: teamRoles){
			if(tr.getTeamName().equals(Constants.GRSF_CATALOGUE_REVIEWER_ROLE))
				reviewers.add(um.getUserById(tr.getUserId()));
		}

		logger.info("List of " + Constants.GRSF_CATALOGUE_REVIEWER_ROLE + " is " + reviewers);

		// build the url that allows to revert the operation TODO
		Operation operation = bean.isMergesInvolved() ? Operation.MERGE : Operation.DISSECT;
		getEncodedUrlManage(operation, username, System.currentTimeMillis(), bean.getKnowledgeBaseIdentifier(), httpServletRequest);
		
		String object = "A GRSF Record has been modified";

		// send the emails reviewers
		String messageReviewer = "";

		// send email to the editor
		String messageEditor = "";
		
		// TODO
	}

	/**
	 * Create the url to be send for reverting the operation
	 * @param httpSession 
	 * @return
	 * @throws Exception 
	 */
	public static String getEncodedUrlManage(Operation operation, String administrator, long timestamp, String uuid, HttpServletRequest httpServletRequest) throws Exception{

		String clientUrl = getCurrentClientUrl(httpServletRequest).split("\\?")[0]; // ignore other parameters		
		RevertOperationUrl operationUrl = new RevertOperationUrl(clientUrl, administrator, timestamp, uuid, operation, operation);
		String shortUrl = operationUrl.getShortUrl();
		logger.info("Encrypted and shortened url " + shortUrl);
		return shortUrl;
	}

	/**
	 * Get the list of hashtags from the actions taken onto the record
	 * @param bean
	 * @return
	 */
	private static List<String> getHashTagsFromActions(ManageProductBean bean) {
		// TODO Auto-generated method stub
		return null;
	}

	/**
	 * Send updates to the knowledge base
	 * @param httpClient 
	 * @param serviceUrl
	 * @param bean
	 * @param catalogue
	 * @param username
	 * @param fullName
	 */
	@SuppressWarnings("unchecked")
	private static void updateKB(CloseableHttpClient httpClient, String serviceUrl, ManageProductBean bean, 
			DataCatalogue catalogue, String username, String fullName) throws Exception{

		JSONObject obj = new JSONObject();
		obj.put(Constants.ADMINISTRATOR_FULLNAME, fullName);
		obj.put(Constants.CATALOGUE_ID, bean.getCatalogueIdentifier());
		obj.put(Constants.KB_ID, bean.getKnowledgeBaseIdentifier());
		obj.put(Constants.NEW_STATUS, bean.getNewStatus().toString().toLowerCase());
		obj.put(Constants.OLD_STATUS, bean.getCurrentStatus().toString().toLowerCase());
		obj.put(Constants.TRACEABILITY_FLAG, bean.isTraceabilityFlag());

		String annotation = bean.getAnnotation();
		if(annotation != null)
			obj.put(Constants.ANNOTATION, annotation.replaceAll("\"", ""));

		obj.put(Constants.SHORT_NAME_OLD, bean.getShortName());

		if(bean.getShortNameUpdated() == null || bean.getShortNameUpdated().isEmpty())
			bean.setShortNameUpdated(bean.getShortName());

		obj.put(Constants.SHORT_NAME_NEW, bean.getShortNameUpdated());
		obj.put(Constants.OLD_STATUS, bean.getCurrentStatus().toString().toLowerCase());

		// prepare connections 
		List<ConnectedBean> connections = bean.getConnectTo();
		JSONArray connectionsJson = new JSONArray();

		for(ConnectedBean c: connections){
			JSONObject cc = new JSONObject();
			if(c.isRemove() || (c.isConnect() && !c.isRemove())){ // do not send it if it needs to be unconnected but not removed
				cc.put(Constants.SOURCE_KNOWLEDGE_BASE_ID, c.getSourceKnowledgeBaseId());
				cc.put(Constants.DEST_KNOWLEDGE_BASE_ID, c.getDestKnowledgeBaseId());
				cc.put(Constants.SOURCE_DOMAIN, c.getSourceDomain());
				cc.put(Constants.CONNECTION_TO_REMOVE, c.isRemove());
			}
			connectionsJson.add(cc);
		}
		obj.put(Constants.CONNECTIONS, connectionsJson);

		// prepare similar grsf records
		List<SimilarGRSFRecord> similarRecords = bean.getSimilarGrsfRecords();
		JSONArray similarRecordsJson = new JSONArray();
		for(SimilarGRSFRecord s: similarRecords){
			JSONObject ss = new JSONObject();
			ss.put(Constants.KB_ID, s.getKnowledgeBaseId());
			ss.put(Constants.MERGE, s.isSuggestedMerge());
			similarRecordsJson.add(ss);
		}
		obj.put(Constants.SIMILAR_GRSF_RECORDS, similarRecordsJson);

		logger.info("Update request looks like " + obj.toJSONString());

		HttpPost request = new HttpPost(serviceUrl + Constants.SERVICE_POST_METHOD);
		request.setHeader("Accept", "application/json");
		request.setHeader("Content-type", "application/json");
		StringEntity params = new StringEntity(obj.toJSONString());
		request.setEntity(params);
		HttpResponse response = httpClient.execute(request);

		logger.debug("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase());

		String result = EntityUtils.toString(response.getEntity());
		JSONParser parser = new JSONParser();
		JSONObject parsedJSON = (JSONObject)parser.parse(result);

		if(response.getStatusLine().getStatusCode() != Constants.STATUS_SUCCESS){
			throw new Exception("Update failed at knowledge base side!");
		}else if(!(boolean) parsedJSON.get(Constants.UPDATE_RESULT))
			throw new IllegalArgumentException(
					"Update failed for the following reason " + parsedJSON.get(Constants.ERROR_MESSAGE));

	}

	/**
	 * Get the scope in which ckan information needs to be discovered from the url
	 * @param httpServletRequest
	 * @return
	 */
	public static String getScopeFromClientUrl(HttpServletRequest httpServletRequest){

		if(httpServletRequest == null)
			throw new IllegalArgumentException("HttpServletRequest is null!");

		String scopeToReturn = null;
		try{
			String clientUrl = getCurrentClientUrl(httpServletRequest).split("\\?")[0];
			logger.debug("Client url is " + clientUrl);

			// check if this information is in session, otherwise set it and return
			HttpSession session = httpServletRequest.getSession();

			if((scopeToReturn = (String) session.getAttribute(clientUrl)) != null){
				logger.debug("Scope to return is " + scopeToReturn);
			}else{
				// ask to the ckan library and set it
				scopeToReturn = ApplicationProfileScopePerUrlReader.getScopePerUrl(clientUrl);
				logger.debug("Scope to return is " + scopeToReturn);
				session.setAttribute(clientUrl, scopeToReturn);
			}
		}catch(Exception e){
			scopeToReturn = getCurrentContext(httpServletRequest, true);
			logger.warn("Failed to determine the scope from the client url, returning the current one: " + scopeToReturn);
		}
		return scopeToReturn;
	}

	/**
	 * Needed to get the url of the client
	 * @param httpServletRequest the httpServletRequest object
	 * @return the instance of the user 
	 * @see the url at client side
	 */
	public static String getCurrentClientUrl(HttpServletRequest httpServletRequest) {
		if(httpServletRequest == null)
			throw new IllegalArgumentException("HttpServletRequest is null!");

		return httpServletRequest.getHeader(Constants.GCUBE_REQUEST_URL);
	}

	/**
	 * Retrieve the current scope by using the portal manager
	 * @param b 
	 * @return a GcubeUser object
	 */
	public static String getCurrentContext(HttpServletRequest request, boolean setInThread){

		if(request == null)
			throw new IllegalArgumentException("HttpServletRequest is null!");

		PortalContext pContext = PortalContext.getConfiguration();
		String context = pContext.getCurrentScope(request);
		logger.debug("Returning context " + context);

		if(context != null && setInThread)
			ScopeProvider.instance.set(context);

		return context;
	}

	/**
	 * Retrieve the current user by using the portal manager
	 * @return a GcubeUser object
	 */
	public static GCubeUser getCurrentUser(HttpServletRequest request){

		if(request == null)
			throw new IllegalArgumentException("HttpServletRequest is null!");

		PortalContext pContext = PortalContext.getConfiguration();
		GCubeUser user = pContext.getCurrentUser(request);
		logger.debug("Returning user " + user);
		return user;
	}

	/**
	 * Given a semantic identifier, check if a record exists and return it
	 * @param suggestedRecordSemanticIdentifier
	 * @param catalogue
	 * @return CkanDataset
	 * @throws Exception in case no record matches the semantic identifier
	 */
	public static CkanDataset getRecordBySemanticIdentifier(
			String suggestedRecordSemanticIdentifier, DataCatalogue catalogue,
			String apiKey) throws Exception {

		if(suggestedRecordSemanticIdentifier == null || suggestedRecordSemanticIdentifier.isEmpty())
			throw new Exception(Constants.GRSF_SEMANTIC_IDENTIFIER_CUSTOM_KEY + " cannot be null or emtpy");

		String query = Constants.GRSF_SEMANTIC_IDENTIFIER_CUSTOM_KEY + "\"" + suggestedRecordSemanticIdentifier+ "\"";
		List<CkanDataset> datasets = catalogue.searchForPackageInOrganization(apiKey, query, 0, 10, Constants.GRSF_ADMIN_ORGANIZATION_NAME);

		if(datasets == null || datasets.isEmpty()){
			String message = "Unable to find dataset with such " + Constants.GRSF_SEMANTIC_IDENTIFIER_CUSTOM_KEY;
			logger.warn(message);
			throw new Exception(message); 
		}

		logger.info("Result size is " + datasets.size());

		if(datasets.size() == 1)
			return datasets.get(0);
		else{

			// worst situation.. we need to check for the right one
			for(CkanDataset dataset: datasets) 
				for(CkanPair extra : dataset.getExtras())
					if(extra.getKey().contains(Constants.GRSF_SEMANTIC_IDENTIFIER_CUSTOM_KEY) && extra.getValue().equals(suggestedRecordSemanticIdentifier)){
						logger.info("Matching dataset has id " + dataset.getId() + " with value " + Constants.GRSF_SEMANTIC_IDENTIFIER_CUSTOM_KEY);
						return dataset;
					}

		}

		// in the end ....
		throw new Exception("Unable to find record with " + Constants.GRSF_SEMANTIC_IDENTIFIER_CUSTOM_KEY + " equals to " + suggestedRecordSemanticIdentifier);

	}

	/**
	 * Exploits the fact that in GRSF the url of a record contains the name (which is unique) of the record itself
	 * @param url
	 * @param clg
	 * @return
	 */
	public static CkanDataset getDatasetFromUrl(String url, DataCatalogue clg, String apiKey){

		if(url == null || url.isEmpty())
			return null;

		// Parse url
		// Create a Pattern object
		Pattern r = Pattern.compile(REGEX_UUID);

		// Now create matcher object.
		Matcher m = r.matcher(url);
		if (m.find()) {
			String uuidFound = m.group();
			logger.debug("Found match for uuid " + uuidFound);
			return clg.getDataset(uuidFound, apiKey);
		}

		return null;
	}

	/**
	 * Exploits the fact that in GRSF the url of a record contains the name (which is unique) of the record itself
	 * @param url
	 * @param clg
	 * @return
	 */
	public static String getDatasetKnowledgeBaseIdFromUrl(String url){

		if(url == null || url.isEmpty())
			return null;

		// Parse url
		// Create a Pattern object
		Pattern r = Pattern.compile(REGEX_UUID);

		// Now create matcher object.
		Matcher m = r.matcher(url);
		if (m.find()) {
			String uuidFound = m.group();
			logger.debug("Found match for uuid " + uuidFound);
			return uuidFound;
		}

		return null;
	}

	//	/**
	//	 * Get extra information to show in the management panel, if any
	//	 * @param extrasAsPairs
	//	 */
	//	public static void getExtrasToShow(List<CkanPair> extrasAsPairs, ){
	//		
	//		Set<String> extrasToShow = getLookedUpExtrasKeys();
	//		if(extrasToShow != null && !extrasToShow.isEmpty()){
	//			Map<String, String> extrasKeyValuePair = new HashMap<String, String>();
	//			 = product.getExtras();
	//			for (CkanPair ckanPair : extrasAsPairs) {
	//				String key = ckanPair.getKey();
	//				String value = ckanPair.getValue();
	//
	//				if(extrasToShow.contains(key)){
	//					String currentValueInMap = extrasKeyValuePair.get(key);
	//					if(currentValueInMap == null)
	//						currentValueInMap = value;
	//					else
	//						currentValueInMap += ", " +  value;
	//					extrasKeyValuePair.put(key, currentValueInMap);
	//				}		
	//			}
	//			toReturn.setExtrasIfAvailable(extrasKeyValuePair);
	//		}
	//		
	//	}

	/**
	 * Get a {@link SimilarGRSFRecord} from a json string
	 * @param json
	 * @return {@link SimilarGRSFRecord}
	 * @throws ParseException 
	 */
	public static SimilarGRSFRecord similarGRSFRecordFromJson(String json) throws ParseException{

		if(json == null)
			return null;

		JSONParser parser = new JSONParser();
		JSONObject object = (JSONObject)parser.parse(json);

		return new SimilarGRSFRecord(
				(String)object.get(Constants.SIMILAR_RECORDS_BEAN_FIELD_DESCRIPTION),
				getDatasetKnowledgeBaseIdFromUrl((String)object.get(Constants.SIMILAR_RECORDS_BEAN_FIELD_URL)),
				(String)object.get(Constants.SIMILAR_RECORDS_BEAN_FIELD_IDENTIFIER),
				(String)object.get(Constants.SIMILAR_RECORDS_BEAN_FIELD_NAME),
				(String)object.get(Constants.SIMILAR_RECORDS_BEAN_FIELD_URL)
				);

	}

	/**
	 * Checks if is into portal.
	 *
	 * @return true, if is into portal
	 */
	public static boolean isIntoPortal() {
		try {
			UserLocalServiceUtil.getService();
			return true;
		}catch (Exception ex) {
			logger.debug("Development Mode ON");
			return false;
		}
	}

	/**
	 * Get Connected bean from record
	 * @param json
	 * @param sourceIdentifier
	 * @param sourceDomain
	 * @return
	 * @throws ParseException
	 */
	public static ConnectedBean connectedBeanRecordFromJson(String json, String sourceIdentifier, String sourceDomain,
			DataCatalogue clg) throws ParseException {

		if(json == null)
			return null;

		JSONParser parser = new JSONParser();
		JSONObject object = (JSONObject)parser.parse(json);

		String uuidDest = (String)object.get(Constants.CONNECTED_RECORD_KNOWLEDGE_BASE_ID_JSON_KEY);
		String url = clg.getUrlFromDatasetIdOrName(uuidDest);
		return new ConnectedBean(
				sourceIdentifier, 
				sourceDomain, 
				(String)object.get(Constants.CONNECTED_RECORD_SHORT_NAME_JSON_KEY), 
				(String)object.get(Constants.CONNECTED_RECORD_SEMANTIC_IDENTIFIER_JSON_KEY), 
				uuidDest,
				url
				);

	}

	/**
	 * Fetch the sysadmin key from the IS
	 * @return
	 * @throws Exception 
	 */
	public static String fetchSysAPI(String context) throws Exception{

		DataCatalogueRunningCluster catalogueRunningInstance = new DataCatalogueRunningCluster(context);
		return catalogueRunningInstance.getSysAdminToken();

	}

}