ckan-metadata-publisher-widget/src/main/java/org/gcube/portlets/widgets/ckandatapublisherwidget/server/CKANPublisherServicesImpl.java

package org.gcube.portlets.widgets.ckandatapublisherwidget.server;

import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import net.htmlparser.jericho.Renderer;
import net.htmlparser.jericho.Segment;
import net.htmlparser.jericho.Source;

import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.folder.FolderItem;
import org.gcube.common.homelibrary.home.workspace.folder.items.GCubeItem;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datacatalogue.ckanutillibrary.CKanUtilsFactory;
import org.gcube.datacatalogue.metadatadiscovery.DataCalogueMetadataFormatReader;
import org.gcube.datacatalogue.metadatadiscovery.bean.MetadataType;
import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataField;
import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataFormat;
import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataValidator;
import org.gcube.datacatalogue.metadatadiscovery.bean.jaxb.MetadataVocabulary;
import org.gcube.portlets.widgets.ckandatapublisherwidget.client.CKanPublisherService;
import org.gcube.portlets.widgets.ckandatapublisherwidget.shared.DatasetMetadataBean;
import org.gcube.portlets.widgets.ckandatapublisherwidget.shared.LicensesBean;
import org.gcube.portlets.widgets.ckandatapublisherwidget.shared.MetaDataProfileBean;
import org.gcube.portlets.widgets.ckandatapublisherwidget.shared.MetaDataTypeWrapper;
import org.gcube.portlets.widgets.ckandatapublisherwidget.shared.MetadataFieldWrapper;
import org.gcube.portlets.widgets.ckandatapublisherwidget.shared.ResourceBean;
import org.gcube.vomanagement.usermanagement.UserManager;
import org.gcube.vomanagement.usermanagement.impl.LiferayUserManager;
import org.gcube.vomanagement.usermanagement.model.GCubeUser;
import org.slf4j.LoggerFactory;

import com.google.gwt.user.server.rpc.RemoteServiceServlet;
import com.liferay.portal.service.UserLocalServiceUtil;

import eu.trentorise.opendata.jackan.CheckedCkanClient;
import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanPair;
import eu.trentorise.opendata.jackan.model.CkanResource;
import eu.trentorise.opendata.jackan.model.CkanTag;

/**
 * Server side of the data publisher.
 * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
 */
@SuppressWarnings("serial")
public class CKANPublisherServicesImpl extends RemoteServiceServlet implements CKanPublisherService{

	// Logger
	private static final org.slf4j.Logger logger = LoggerFactory.getLogger(CKANPublisherServicesImpl.class);

	/**
	 * Get the current scope (/gcube is returned if in development)
	 * @return
	 */
	private String getCurrentScope(){

		if(isWithinPortal())
			return ScopeProvider.instance.get();
		else
			return "/gcube"; // Development

	}

	/**
	 * Return the ckan catalogue url for this scope.
	 * @return
	 */
	private String getCatalogueUrl(){

		logger.info("Request for catalogue url");

		try{
			String currentScope = getCurrentScope();
			return CKanUtilsFactory.getInstance().getCkanUtilsForScope(currentScope).getCatalogueUrl();
		}catch(Exception e){
			logger.error("Failed to retrieve catalogue url information", e);
		}

		return null;
	}

	/**
	 * Retrieve the api key for this user
	 * @param owner
	 * @return
	 */
	private String getCKANApikeyFromUser(String owner) {

		logger.info("Request for user api key");

		try{
			String currentScope = getCurrentScope();
			return CKanUtilsFactory.getInstance().getCkanUtilsForScope(currentScope).getApiKeyFromUser(owner);
		}catch(Exception e){
			logger.error("Failed to retrieve api key for user " + owner, e);
		}

		return null;
	}

	/**
	 * Retrieve the list of organizations in which the user can publish
	 * @param owner
	 * @return
	 */
	private List<String> getUserOrganizationsList(String owner) {

		logger.info("Request for user " + owner + " organizations list");

		try{

			String currentScope = getCurrentScope();
			return CKanUtilsFactory.getInstance().getCkanUtilsForScope(currentScope).getOrganizationsNamesByUser(owner);

		}catch(Exception e){
			logger.error("Failed to retrieve user's organizations", e);
		}

		return null;
	}

	/**
	 * Online or in development mode?
	 * @return true if you're running into the portal, false if in development
	 */
	private boolean isWithinPortal() {
		try {
			UserLocalServiceUtil.getService();
			return true;
		} 
		catch (com.liferay.portal.kernel.bean.BeanLocatorException ex) {			
			logger.trace("Development Mode ON");
			return false;
		}			
	}

	/**
	 * Find a license id given the license text.
	 * @param chosenLicense
	 * @return
	 */
	private String findLicenseIdByLicense(String chosenLicense) {

		String currentScope = getCurrentScope();
		try {
			return CKanUtilsFactory.getInstance().getCkanUtilsForScope(currentScope).findLicenseIdByLicense(chosenLicense);
		} catch (Exception e) {
			logger.error("Failed to retrieve license id", e);
		}

		return null;
	}

	/**
	 * Retrieve the list of metadata beans
	 * @return
	 */
	public List<MetaDataProfileBean> getMetadataProfilesList() {

		List<MetaDataProfileBean> beans = new ArrayList<MetaDataProfileBean>();

		try {

			ScopeProvider.instance.set("/gcube/devsec/devVRE"); // TODO remove
			DataCalogueMetadataFormatReader reader = new DataCalogueMetadataFormatReader();

			for (MetadataType mt : reader.getListOfMetadataTypes()) {
				MetadataFormat metadata = reader.getMetadataFormatForMetadataType(mt);

				// we need to wrap the list of metadata
				List<MetadataFieldWrapper> wrapperList = new ArrayList<MetadataFieldWrapper>();
				List<MetadataField> toWrap = metadata.getMetadataFields();
				for(MetadataField metadataField: toWrap){

					MetadataFieldWrapper wrapperObj = new MetadataFieldWrapper();
					wrapperObj.setDefaulValue(metadataField.getDefaulValue());
					wrapperObj.setFieldName(metadataField.getFieldName());
					wrapperObj.setIsBoolean(metadataField.getIsBoolean());
					wrapperObj.setMandatory(metadataField.getMandatory());
					wrapperObj.setNote(metadataField.getNote());

					MetadataValidator validator = metadataField.getValidator();
					if(validator != null)
						wrapperObj.setValidator(validator.getRegularExpression());

					MetadataVocabulary vocabulary = metadataField.getVocabulary();
					if(vocabulary != null)
						wrapperObj.setVocabulary(vocabulary.getVocabularyFields());

					// add to the list
					wrapperList.add(wrapperObj);

				}
				
				// wrap the mt as well
				MetaDataTypeWrapper typeWrapper = new MetaDataTypeWrapper();
				typeWrapper.setDescription(mt.getDescription());
				typeWrapper.setId(mt.getId());
				typeWrapper.setName(mt.getName());
				MetaDataProfileBean bean = new MetaDataProfileBean(typeWrapper, wrapperList);
				beans.add(bean);
			}

			logger.debug("List of beans is " + beans);
			return beans;

		} catch (Exception e) {
			logger.error("Error while retrieving metadata beans ", e);
		}

		return null;
	}

	@Override
	public LicensesBean getLicenses() {

		logger.info("Request for CKAN licenses");
		try {
			String currentScope = getCurrentScope();
			List<String> titlesLicenses;
			titlesLicenses = CKanUtilsFactory.getInstance().getCkanUtilsForScope(currentScope).getLicenseTitles();
			// return the bean
			return new LicensesBean(titlesLicenses);
		} catch (Exception e) {
			logger.error("Failed to retrieve licenses", e);
		}
		return null;
	}

	@Override
	public DatasetMetadataBean getDatasetBean(String folderId, String owner){

		DatasetMetadataBean bean = null;

		logger.info("Bean request for " + folderId + " and " + owner);

		if(isWithinPortal()){
			try{

				logger.debug("Request dataset metadata bean for folder with id " + folderId
						+ " whose owner is " + owner);
				// get usermanager (liferay)
				UserManager liferUserManager = new LiferayUserManager();
				GCubeUser userOwner = liferUserManager.getUserByUsername(owner);

				// build bean
				logger.debug("Building bean");
				bean = new DatasetMetadataBean();

				bean.setId(folderId);
				bean.setOwnerIdentifier(owner);
				bean.setVersion(1);
				bean.setAuthor(userOwner.getFullname());
				bean.setAuthorEmail(userOwner.getEmail());
				bean.setMaintainer(userOwner.getFullname());
				bean.setMaintainerEmail(userOwner.getEmail());
				bean.setOrganizationList(getUserOrganizationsList(owner));

				// if the request comes from the workspace
				if(folderId != null && !folderId.isEmpty()){


					Workspace ws = HomeLibrary
							.getHomeManagerFactory()
							.getHomeManager()
							.getHome(owner).getWorkspace();

					WorkspaceItem retrievedItem = ws.getItem(folderId);

					// set some info
					bean.setTitle(retrievedItem.getName());
					bean.setDescription(retrievedItem.getDescription());

					// retrieve gcube items of the folder
					Map<String, String> folderItems = getGcubeItemProperties(retrievedItem);
					bean.setCustomFields(folderItems);

					// check the resources within the folder (skip subdirectories)
					List<String> childrenIds = new ArrayList<String>();

					for (WorkspaceItem file : retrievedItem.getChildren()) {
						if(!file.isFolder()) // ok, it's a file
							childrenIds.add(file.getId());
					}

					bean.setResourcesIds(childrenIds);

					// retrieve the metadata
					List<MetaDataProfileBean> metadataBeans = getMetadataProfilesList();
					bean.setMetadataList(metadataBeans);

				}

			}catch(Exception e){
				logger.error("Error while retrieving folder information", e);
			}
		}else{

			try{

				bean = new DatasetMetadataBean();
				bean.setId(folderId);
				bean.setDescription("This is a fantastic description");
				bean.setVersion(1);
				bean.setTitle("dataset-" + Calendar.getInstance().getTimeInMillis());
				bean.setAuthor("Costantino Perciante");
				bean.setAuthorEmail("costantino.perciante@isti.cnr.it");
				bean.setMaintainer("Costantino Perciante");
				bean.setMaintainerEmail("costantino.perciante@isti.cnr.it");
				bean.setOrganizationList(getUserOrganizationsList(owner));
				bean.setOwnerIdentifier(owner);

				if(folderId != null && !folderId.isEmpty()){
					ScopeProvider.instance.set("/gcube");

					Workspace ws = HomeLibrary
							.getHomeManagerFactory()
							.getHomeManager()
							.getHome(owner).getWorkspace();

					WorkspaceItem retrievedItem = ws.getItem(folderId);

					// retrieve gcube items of the folder
					Map<String, String> folderItems = getGcubeItemProperties(retrievedItem);
					bean.setCustomFields(folderItems);

					// check the resources within the folder (skip subdirectories)
					List<String> childrenIds = new ArrayList<String>();

					for (WorkspaceItem file : retrievedItem.getChildren()) {
						if(!file.isFolder()) // ok, it's a file
							childrenIds.add(file.getId());
					}

					bean.setResourcesIds(childrenIds);
				}

				// retrieve the metadata
				List<MetaDataProfileBean> metadataBeans = getMetadataProfilesList();
				bean.setMetadataList(metadataBeans);

			}catch(Exception e){
				logger.error("Error while building bean into dev mode", e);
			}
		}

		return bean;
	}

	/** Gets the gcube item properties.
	 *
	 * @param item the item
	 * @return the gcube item properties
	 */
	private Map<String, String> getGcubeItemProperties(WorkspaceItem item) {

		if(item instanceof GCubeItem){
			GCubeItem gItem = (GCubeItem) item;
			try {
				if(gItem.getProperties()!=null){
					Map<String, String> map = gItem.getProperties().getProperties();
					HashMap<String, String> properties = new HashMap<String, String>(map.size()); //TO PREVENT GWT SERIALIZATION ERROR
					for (String key : map.keySet())
						properties.put(key, map.get(key));

					return properties;
				}
			} catch (InternalErrorException e) {
				logger.error("Error in server getItemProperties: ", e);
				return null;
			}
		}
		return null;
	}

	@Override
	public String createCKanDataset(DatasetMetadataBean toCreate, boolean isWorkspaceRequest) {

		// retrieve ckan's catalog url
		String ckanPortalUrl = getCatalogueUrl();

		// retrieve the owner identifier
		String owner = toCreate.getOwnerIdentifier();

		// retrieve the api key for this user
		String apiKey = getCKANApikeyFromUser(owner);

		logger.info("Trying to create the dataset described by this bean " + toCreate + " into CKAN catalog at url " + ckanPortalUrl);

		// Checked client
		CheckedCkanClient client = new CheckedCkanClient(ckanPortalUrl, apiKey);

		// create the base dataset and fill it
		CkanDataset dataset = new CkanDataset();

		// set dataset info (same id as the folder)
		dataset.setId(toCreate.getId());

		// get the name from the title
		dataset.setName(nameFromTitle(toCreate.getTitle()));
		dataset.setTitle(toCreate.getTitle());

		CkanOrganization orgTest = client.getOrganization(toCreate.getSelectedOrganization());
		dataset.setOwnerOrg(orgTest.getId());
		dataset.setAuthor(toCreate.getAuthor());
		dataset.setAuthorEmail(toCreate.getAuthorEmail());
		dataset.setMaintainer(toCreate.getMaintainer());
		dataset.setMaintainerEmail(toCreate.getMaintainerEmail());
		dataset.setVersion(String.valueOf(toCreate.getVersion()));

		// description must be escaped
		Source description = new Source(toCreate.getDescription());
		Segment htmlSeg = new Segment(description, 0, description.length());
		Renderer htmlRend = new Renderer(htmlSeg);
		dataset.setNotes(htmlRend.toString());

		logger.debug("Description (escaped is ) " + htmlRend.toString());

		// visibility TODO
		dataset.setOpen(toCreate.getVisibility());

		// iterate over the licenses to find the id of the chosen one
		String chosenLicense = toCreate.getLicense();
		String licenseId = findLicenseIdByLicense(chosenLicense);
		dataset.setLicenseId(licenseId);

		// set the tags, if any
		if(toCreate.getTags() != null && !toCreate.getTags().isEmpty()){

			// convert to ckan tags
			List<CkanTag> ckanTags = new ArrayList<CkanTag>(toCreate.getTags().size());
			for (String stringTag : toCreate.getTags()) {
				ckanTags.add(new CkanTag(stringTag));
			}

			dataset.setTags(ckanTags);
		}

		// set the custom fields, if any
		if(toCreate.getCustomFields() != null && !toCreate.getCustomFields().isEmpty()){

			// iterate and create 
			Iterator<Entry<String, String>> iterator = toCreate.getCustomFields().entrySet().iterator();

			List<CkanPair> extras = new ArrayList<CkanPair>(toCreate.getCustomFields().entrySet().size());

			while (iterator.hasNext()) {

				Map.Entry<String, String> entry = (Map.Entry<String, String>) iterator.next();
				extras.add(new CkanPair(entry.getKey(), entry.getValue()));

			}

			dataset.setExtras(extras);
		}

		// check if we need to add the resources
		if(toCreate.isAddResources() && isWorkspaceRequest){

			logger.debug("We need to add resources to the dataset");

			try{

				ScopeProvider.instance.set("/gcube");
				Workspace ws = HomeLibrary
						.getHomeManagerFactory()
						.getHomeManager()
						.getHome(toCreate.getOwnerIdentifier()).getWorkspace();

				List<CkanResource> resources = new ArrayList<CkanResource>();


				for(String resourceId: toCreate.getResourcesIds()){

					FolderItem item = (FolderItem)ws.getItem(resourceId);
					CkanResource newResource = new CkanResource();
					newResource.setDescription(item.getDescription());
					newResource.setId(item.getId());
					newResource.setUrl(item.getPublicLink(false));
					newResource.setName(item.getName());
					newResource.setMimetype(item.getMimeType());

					resources.add(newResource);
				}

				// add to the dataset
				dataset.setResources(resources);

			}catch(Exception e){
				logger.error("Unable to add those resources to the dataset", e);
			}

		}

		// try to create
		CkanDataset res = null;
		try{

			res = client.createDataset(dataset);

		}catch(Exception e){

			// try to update
			logger.error("Error while creating the dataset, probably it already exists.", e);

		}

		if(res != null)
			logger.debug("Dataset created/updated " + res.getId());
		else{
			logger.error("Dataset described by " + toCreate + " not created!");
			return null;
		}

		return res.getId();
	}

	/**
	 * Generate the catalogue's dataset name from its title
	 * @param title
	 * @return
	 */
	private String nameFromTitle(String title) {
		String convertedName = title.replaceAll(" ", "-");
		convertedName = convertedName.toLowerCase();
		if(convertedName.endsWith("-"))
			convertedName  = convertedName.substring(0, convertedName.length() - 2);

		return convertedName;
	}

	/**
	 * Utility methods
	 * @param URLName
	 * @return
	 */
	private static boolean exists(String URLName){
		try {
			HttpURLConnection.setFollowRedirects(true);
			HttpURLConnection con = (HttpURLConnection) new URL(URLName).openConnection();
			con.setRequestMethod("HEAD");
			logger.debug("Return code is " + con.getResponseCode());
			return (con.getResponseCode() == HttpURLConnection.HTTP_OK);
		}
		catch (Exception e) {
			logger.error("Exception while checking url", e);
			return false;
		}
	}

	@Override
	public ResourceBean addResourceToDataset(ResourceBean resourceBean, String datasetId, String owner) {

		logger.debug("Incoming request for creating new resource for dataset with id " + datasetId);
		logger.debug("Owner is " + owner + " and resource is " + resourceBean);

		// of course, if it exists
		if(exists(resourceBean.getUrl())){

			try{
				// retrieve ckan's catalog url
				String ckanPortalUrl = getCatalogueUrl();

				// retrieve the api key for this user
				String apiKey = getCKANApikeyFromUser(owner);

				CkanResource resource = new CkanResource(ckanPortalUrl, datasetId);
				resource.setName(resourceBean.getName());

				// escape description
				Source description = new Source(resourceBean.getDescription());
				Segment htmlSeg = new Segment(description, 0, description.length());
				Renderer htmlRend = new Renderer(htmlSeg);

				resource.setDescription(htmlRend.toString());
				resource.setUrl(resourceBean.getUrl());
				resource.setOwner(owner);

				// Checked client
				CheckedCkanClient client = new CheckedCkanClient(ckanPortalUrl, apiKey);
				CkanResource createdRes = client.createResource(resource);

				if(createdRes != null){

					logger.debug("Resource  " + createdRes.getName() + " is now available");

					// set its id and turn it to the client
					resourceBean.setId(createdRes.getId());
					return resourceBean;

				}

			}catch(Exception e){
				logger.error("Unable to create new resource", e);
			}
		}

		logger.debug("No resource created");

		return null;
	}

	@Override
	public boolean deleteResourceFromDataset(ResourceBean resource,
			String owner) {
		logger.debug("Request for deleting resource " + resource);

		try{

			// retrieve ckan's catalog url
			String ckanPortalUrl = getCatalogueUrl();

			// retrieve the api key for this user
			String apiKey = getCKANApikeyFromUser(owner);

			// Checked client
			CheckedCkanClient client = new CheckedCkanClient(ckanPortalUrl, apiKey);
			client.deleteResource(resource.getId());

			return true;

		}catch(Exception e){
			logger.error("Unable to delete such resource", e);
		}

		return false;
	}

}