package org.gcube.datacatalogue.ckanutillibrary.test; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.datacatalogue.ckanutillibrary.server.ApplicationProfileScopePerUrlReader; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueFactory; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueImpl; import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean; import org.junit.Before; import org.junit.Test; import org.slf4j.LoggerFactory; import eu.trentorise.opendata.jackan.model.CkanOrganization; import eu.trentorise.opendata.jackan.model.CkanUser; /** * The Class TestDataCatalogueLib. * * @author Francesco Mangiacrapa at ISTI-CNR Pisa (Italy) * Jun 1, 2020 */ public class TestDataCatalogueLib { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(TestDataCatalogueLib.class); private DataCatalogueFactory factory; private String scope = "/gcube/devsec/devVRE"; //private String scope = "/pred4s/preprod/preVRE"; //private String scope = "/d4science.research-infrastructures.eu/SoBigData/Catalogue-TerritoriAperti"; //private String scope = "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab"; //private String testUser = "costantino_perciante"; private String testUser = "francesco.mangiacrapa"; private String authorizationToken = "8e74a17c-92f1-405a-b591-3a6090066248-98187548"; //devVRE String subjectId = "aa_father4"; String objectId = "bb_son4"; /** * Before. * * @throws Exception the exception */ @Before public void before() throws Exception{ factory = DataCatalogueFactory.getFactory(); } /** * Factory test. * * @throws Exception the exception */ //@Test public void factoryTest() throws Exception{ DataCatalogueFactory factory = DataCatalogueFactory.getFactory(); while(true){ factory.getUtilsPerScope(scope); Thread.sleep(60* 1000 * 3); factory.getUtilsPerScope(scope); break; } for (int i = 0; i < 5; i++) { Thread.sleep(1000); factory.getUtilsPerScope(scope); } } /** * Gets the scope per url. * * @return the scope per url */ //@Test public void getScopePerUrl(){ ScopeProvider.instance.set(scope); String url = "https://dev4.d4science.org/group/devvre/ckan"; String scopeToUse = ApplicationProfileScopePerUrlReader.getScopePerUrl(url); logger.debug("Retrieved scope is " + scopeToUse); ScopeProvider.instance.reset(); // the following sysout should print null String url2 = "https://dev4.d4science.org/group/devvre/ckan"; String scopeToUse2 = ApplicationProfileScopePerUrlReader.getScopePerUrl(url2); logger.debug("Retrieved scope is " + scopeToUse2); } /** * Gets the user role by group. * * @return the user role by group * @throws Exception the exception */ //@Test public void getUserRoleByGroup() throws Exception{ DataCatalogueImpl instance = factory.getUtilsPerScope(scope); String username = testUser; long init = System.currentTimeMillis(); instance.getUserRoleByGroup(username); long end = System.currentTimeMillis(); logger.debug("Time taken " + (end - init)); } /** * Gets the user role by organization. * * @return the user role by organization * @throws Exception the exception */ //@Test public void getUserRoleByOrganization() throws Exception{ DataCatalogueImpl instance = factory.getUtilsPerScope(scope); String username = testUser; long init = System.currentTimeMillis(); instance.getUserRoleByOrganization(username); long end = System.currentTimeMillis(); logger.debug("Time taken " + (end - init)); } //@Test public void getRoleOfUserInOrganization() throws Exception{ String orgName = "devvre"; DataCatalogueImpl utils = factory.getUtilsPerScope(scope); CkanOrganization org = utils.getOrganizationByIdOrName(orgName); String role = utils.getRoleOfUserInOrganization(testUser, org.getName()); logger.debug("The user "+testUser+" in the org "+org.getName() + " has the role "+role); } //@Test public void getOrganizationForName() throws Exception{ String orgName = "devvre"; DataCatalogueImpl utils = factory.getUtilsPerScope(scope); CkanOrganization org = utils.getOrganizationByIdOrName(orgName); logger.debug("The "+CkanOrganization.class.getSimpleName()+" is: "+org.getName()); logger.debug("LandingPages of "+CkanOrganization.class.getSimpleName()+" for name " + utils.getLandingPages()); if(org.getUsers()!=null) { for (CkanUser user : org.getUsers()) { logger.debug("User: "+user.getName()); } } } //@Test public void getLandingPages() throws Exception{ DataCatalogueImpl utils = factory.getUtilsPerScope(scope); logger.debug("Landing pages " + utils.getLandingPages()); } //@Test public void countDatasetForOganization() throws Exception{ DataCatalogueImpl utils = new DataCatalogueImpl(scope,true); List listOrgIds = utils.getOrganizationsIds(); System.out.println("'ORGANIZATION NAME': 'NUMBER OF PACKAGE'"); for (String orgID : listOrgIds) { CkanOrganization org = utils.getOrganizationByIdOrName(orgID); int packageCount = org.getPackageCount(); //logger.info("ORG: "+org.getName() + " has "+packageCount+ " package/s"); System.out.println(org.getName()+": "+packageCount); } } @Test public void createDataset() throws Exception{ try { ScopeProvider.instance.set(scope); SecurityTokenProvider.instance.set(authorizationToken); DataCatalogueImpl instance = factory.getUtilsPerScope(scope); String licenseId = instance.getLicenses().get(0).getId(); Map> customFieldsMultiple = new HashMap>(); for (int i = 0; i < 10; i++) { List values = new ArrayList(); for (int j = 0; j < new Random().nextInt(3); j++) { values.add("value-random-"+ new Random().nextInt(10)); } customFieldsMultiple.put("key-random-"+new Random().nextInt(10), values); } boolean setSearchable = true; boolean setPublic = false; List resources = null; String orgName = scope.split("/")[3].toLowerCase(); //System.out.println("Org name by VRE: "+orgName); logger.debug("Org name by VRE: "+orgName); int random = new Random().nextInt(); String datasetTitle = "a dataset created by catalogue-util-library "+random; String datasetName = datasetTitle.replace(" ", ""); String createdDataset = instance.createCKanDatasetMultipleCustomFields( testUser, "a dataset created by catalogue-util-library "+random, datasetName, orgName, testUser, null, null, null, 1, "the description", licenseId, Arrays.asList("tag 1","tag 2","tag 3"), customFieldsMultiple, resources, setPublic, setSearchable); logger.info(createdDataset); }catch (Exception e) { e.printStackTrace(); } } // //@Test // public void getSysadminEmail() throws Exception{ // DataCatalogueImpl utils = factory.getUtilsPerScope(scope); // System.out.println(utils.getCatalogueEmail()); // } // // //@Test // public void getStatistics() throws Exception{ // // DataCatalogueImpl utils = factory.getUtilsPerScope(scope); // logger.debug("Statistics " + utils.getStatistics()); // // } // // // // // //@Test // public void getGroupForName() throws Exception{ // // String groupName = "ariadne"; // // DataCatalogueImpl utils = factory.getUtilsPerScope(scope); // // CkanGroup org = utils.getGroupByName(groupName); // // logger.debug("The "+CkanGroup.class.getSimpleName()+" is: "+org.getName()); // logger.debug("LandingPages of "+CkanGroup.class.getSimpleName()+" for name " + utils.getLandingPages()); // if(org.getUsers()!=null) { // // for (CkanUser user : org.getUsers()) { // logger.debug("User: "+user.getName()); // } // } // } // // //@Test // public void getDatasetIdsFromDB() throws Exception{ // DataCatalogueImpl utils = factory.getUtilsPerScope(scope); // List ids = utils.getProductsIdsInGroupOrOrg("aquamaps", true, 0, Integer.MAX_VALUE); // logger.debug("Size is " + ids.size()); // } // // //@Test // public void searchInOrganization() throws Exception{ // DataCatalogueImpl utils = factory.getUtilsPerScope(scope); // String apiKey = utils.getApiKeyFromUsername(testUser); // List matches = utils.searchForPackageInOrganization(apiKey, "\"asfis:HMC+eez:AGO;FAO+grsf-org:INT+eez:AGO;RFB+iso3:AGO+isscfg:01.1.1\"", 0, 10, "grsf_admin"); // logger.debug("Size is " + matches.size()); // } // // //@Test // public void search() throws Exception{ // DataCatalogueImpl utils = factory.getUtilsPerScope(scope); // String apiKey = utils.getApiKeyFromUsername(testUser); // List matches = utils.searchForPackage(apiKey, "\"asfis:HMC+eez:AGO;FAO+grsf-org:INT+eez:AGO;RFB+iso3:AGO+isscfg:01.1.1\"", 0, 10); // logger.debug("Size is " + matches.size()); // } // // // @Test // public void testManageProduct() throws Exception{ // // DataCatalogueImpl catalogue = factory.getUtilsPerScope(scope); // String apiKey = catalogue.getApiKeyFromUsername("costantino_perciante"); // //Map> map = new HashMap>(); // //map.put("a new custom field", Arrays.asList("a new custom field 2")); // //catalogue.patchProductCustomFields("test-searchable-504043", apiKey, map); // catalogue.removeCustomField("test-searchable-504043", "a new custom field", "a new custom field", apiKey); // } // // //@Test // public void getRole() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // instance.getOrganizationsAndRolesByUser(testUser, Arrays.asList(RolesCkanGroupOrOrg.ADMIN, RolesCkanGroupOrOrg.EDITOR, RolesCkanGroupOrOrg.MEMBER // )); // // } // // //@Test // public void datasetsRelationshipCreate() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // // DatasetRelationships relation = DatasetRelationships.parent_of; // // boolean resC = instance.createDatasetRelationship(subjectId, objectId, relation, "Comment for this relationship", instance.getApiKeyFromUsername(testUser)); // // logger.debug("Res is " + resC); // } // // //@Test // public void datasetsRelationshipDelete() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // DatasetRelationships relation = DatasetRelationships.child_of; // // boolean resD = instance.deleteDatasetRelationship(subjectId, objectId, relation, instance.getApiKeyFromUsername(testUser)); // // logger.debug("ResD is " + resD); // } // // //@Test // public void datasetRelationshipRetrieve() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // // List res = instance.getRelationshipDatasets(subjectId, objectId, instance.getApiKeyFromUsername(testUser)); // // logger.debug("Relationships " + res); // // } // // // //@Test // public void testgetApiKeyFromUser() throws Exception { // // logger.debug("Testing getApiKeyFromUser"); // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // // String username = "francescomangiacrapa"; // String key = instance.getApiKeyFromUsername(username); // // System.out.println("key for " + username + " is " + key); // } // // //@Test // public void testgetUserFromApiKey() throws Exception { // // logger.debug("Testing getApiKeyFromUser"); // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // // String key = "put-your-key-here"; // CKanUserWrapper user = instance.getUserFromApiKey(key); // // System.out.println("user for " + key + " is " + user); // } // // //@Test // public void getOrganizationsByUser() throws Exception { // // System.out.println("Testing getOrganizationsByUser"); // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // // String username = "francescomangiacrapa"; // List organizations = instance.getOrganizationsByUser(username); // // System.out.println("organizations for user " + username + " are: "); // // for (CkanOrganization ckanOrganization : organizations) { // System.out.println("-" + ckanOrganization.getName()); // } // } // // //@Test // public void getGroupsAndRolesByUser() throws Exception { // // logger.debug("Testing getGroupsAndRolesByUser"); // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // // String username = "andrea.rossi"; // instance = new DataCatalogueImpl("/gcube"); // List rolesToMatch = new ArrayList(); // rolesToMatch.add(RolesCkanGroupOrOrg.ADMIN); // rolesToMatch.add(RolesCkanGroupOrOrg.MEMBER); // rolesToMatch.add(RolesCkanGroupOrOrg.EDITOR); // Map> map = instance.getOrganizationsAndRolesByUser(username, rolesToMatch); // // System.out.println("organizations for user " + username + " are " + map); // } // // //@Test // public void getUsers() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // // List rolesToMatch = new ArrayList(); // rolesToMatch.add(RolesCkanGroupOrOrg.ADMIN); // rolesToMatch.add(RolesCkanGroupOrOrg.EDITOR); // // Map> orgs = instance.getOrganizationsAndRolesByUser("costantino_perciante", rolesToMatch); // // Iterator>> iterator = orgs.entrySet().iterator(); // // while (iterator.hasNext()) { // Map.Entry> entry = (Map.Entry>) iterator // .next(); // // logger.debug("Org is " + entry.getKey() + " and role is " + entry.getValue().get(0)); // // } // } // // // // //@Test // public void createUsers() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope("/gcube/devsec/devVRE"); // CheckedCkanClient client = new CheckedCkanClient(instance.getCatalogueUrl(), instance.getApiKeyFromUsername("costantino_perciante")); // CkanUser editorUser = new CkanUser("user_editor_devvre", "user_editor_devvre@test.it", ""); // client.createUser(editorUser); // CkanUser adminUser = new CkanUser("user_admin_devvre", "user_admin_devvre@test.it", ""); // client.createUser(adminUser); // CkanUser memberUser = new CkanUser("user_member_devvre", "user_member_devvre@test.it", ""); // client.createUser(memberUser); // // } // // //@Test // public void createAsEditor() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope("/gcube/devsec/devVRE"); // boolean checkedEditor = instance.checkRoleIntoOrganization("user_editor_devvre", "devvre", RolesCkanGroupOrOrg.MEMBER); // if(checkedEditor){ // logger.debug("Created editor in devvre? " + checkedEditor); // } // boolean checkedAdmin = instance.checkRoleIntoOrganization("user_admin_devvre", "devvre", RolesCkanGroupOrOrg.MEMBER); // if(checkedAdmin){ // logger.debug("Created admin in devvre? " + checkedAdmin); // } // boolean checkedMember = instance.checkRoleIntoOrganization("user_member_devvre", "devvre", RolesCkanGroupOrOrg.MEMBER); // if(checkedMember){ // logger.debug("Created member in devvre? " + checkedMember); // } // } // // // //@Test // public void adminCreateDataset() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope("/gcube/devsec/devVRE"); // //instance.createCKanDataset(instance.getApiKeyFromUsername("user_admin_devvre"), "dataset_as_admin_devvre", "devvre", null, null, null, null, 1, null, null, null, null, null, false); // instance.createCKanDataset(instance.getApiKeyFromUsername("user_admin_devvre"), "dataset_as_admin_devvre_private", null, "devvre", null, null, null, null, 1, null, null, null, null, null, false); // } // // //@Test // public void adminChangeVisibility() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope("/gcube/devsec/devVRE"); // //instance.createCKanDataset(instance.getApiKeyFromUsername("user_editor_devvre"), "dataset_as_editor_devvre", "devvre", null, null, null, null, 1, null, null, null, null, null, false); // //instance.setDatasetPrivate(true, "3571cca5-b0ae-4dc6-b791-434a8e062ce5", "dataset_as_admin_devvre_public", instance.getApiKeyFromUsername("user_admin_devvre")); // // boolean res = instance.setDatasetPrivate(false, "3571cca5-b0ae-4dc6-b791-434a8e062ce5", "33bbdcb1-929f-441f-8718-a9e5134f517d", instance.getApiKeyFromUsername("user_admin_devvre")); // logger.debug(""+res); // // // CheckedCkanClient client = new CheckedCkanClient(instance.getCatalogueUrl(), instance.getApiKeyFromUsername("user_admin_devvre")); // // CkanDataset dataset = client.getDataset("dataset_as_admin_devvre_private"); // // logger.debug("Current value for private: " + dataset.isPriv()); // // dataset.setPriv(!dataset.isPriv()); // // CkanDataset datasetUpd = client.updateDataset(dataset); // // logger.debug("Private value is " + datasetUpd.isPriv()); // // // // } // // //@Test // public void testInvalidOrgRole() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // boolean res = instance.isRoleAlreadySet("francesco_mangiacrapa", "devvre_group", RolesCkanGroupOrOrg.ADMIN, true); // logger.debug(""+res); // // // set to editor // instance.checkRoleIntoGroup("francesco_mangiacrapa", "devvre_group", RolesCkanGroupOrOrg.EDITOR); // // } // // //@Test // public void existProductWithNameOrId() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // boolean res = instance.existProductWithNameOrId("notification_portlet_2"); // logger.debug(""+res); // // } // // //@Test // public void testGroupAssociation() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // String datasetId = "test_grsf_group_stock_groups_sadsad"; // String groupName = "assessment-unit"; // instance.assignDatasetToGroup(groupName, datasetId, instance.getApiKeyFromUsername("costantino_perciante")); // // } // // //@Test // public void testAddResource() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // String datasetId = "test_publish_folder_15_44"; // //instance.assignDatasetToGroup(groupName, datasetId, instance.getApiKeyFromUsername("costantino_perciante")); // // String api = instance.getApiKeyFromUsername("costantino_perciante"); // CheckedCkanClient client = new CheckedCkanClient(instance.getCatalogueUrl(), api); // List randomName = Arrays.asList("FIRMS", "RAM", "FishSource"); // for (int i = 0; i < 100; i++) { // // CkanResource resource = new CkanResource("https://goo.gl/FH5AQ5", datasetId); // String name = randomName.get((int)Math.round(Math.ceil(Math.random() * 3))); // // resource.setName(name); // client.createResource(resource); // // } // } // // //@Test // public void checkGroupRole() throws Exception{ // // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // String role = instance.getRoleOfUserInGroup("francesco.mangiacrapa", "test-by-francesco", instance.getApiKeyFromUsername("francesco.mangiacrapa")); // logger.debug("Role is " + role); // } // // //@Test // public void getURL() throws Exception{ // DataCatalogueImpl util = factory.getUtilsPerScope("/gcube/devNext/NextNext"); // // CkanDataset dataset = new CkanDataset(); // CheckedCkanClient client = new CheckedCkanClient(util.getCatalogueUrl(), util.getApiKeyFromUsername("costantino_perciante")); // // dataset.setName("random-test-" + UUID.randomUUID().toString().substring(0, 5)); // dataset.setOwnerOrg(client.getOrganization("devvre").getId()); // dataset.setOpen(true); // dataset.setPriv(true); // //CkanDataset id = client.createDataset(dataset); // // // util.setDatasetPrivate(true, client.getOrganization("devvre").getId(), id.getId(), util.getApiKeyFromUsername("costantino_perciante")); // // } // // //@Test // public void deleteAndPurgeDataset() throws Exception{ // // DataCatalogueImpl util = factory.getUtilsPerScope("/gcube/devNext/NextNext"); // boolean deleted = util.deleteProduct("random-test-56sadadsfsdf", util.getApiKeyFromUsername("user_admin_devvre"), true); // logger.debug("Deleted ? " + deleted); // // } // // // @Test // public void testSearchableSet() throws Exception{ // // DataCatalogueImpl util = factory.getUtilsPerScope("/gcube/devNext/NextNext"); // boolean setSearchability = util.setSearchableField("1b261d07-9f9c-414f-ad8d-c5aa429548fc", false); // logger.debug("Searchability set? " + setSearchability); // } // // //@Test // public void testNameConversion(){ // // logger.debug(CatalogueUtilMethods.fromCKanUsernameToUsername("costantino_perciante")); // // } // // //@Test // public void getGroupsRoles() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // Map> res = instance.getRolesAndUsersGroup("abundance-level"); // logger.debug(res.toString()); // // } // // //@Test // public void testResourcePatch() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // String id = "858f5e77-80c2-4cb2-bcfc-77529693dc9a"; // instance.patchResource(id, "http://ftp.d4science.org/previews/69cc0769-de6f-45eb-a842-7be2807e8887.jpg", "new_name_for_testing_patch.csv", "description test", "", instance.getApiKeyFromUsername("costantino_perciante")); // } // // //@Test // public void testPatchProduct() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // Map> customFieldsToChange = new HashMap>(); // customFieldsToChange.put("Status", Arrays.asList("Pending")); // instance.patchProductCustomFields("a-test-to-ignore", instance.getApiKeyFromUsername("costantino_perciante"), customFieldsToChange, false); // } // // //@Test // public void addTag()throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // instance.addTag("test-after-tags-editing", instance.getApiKeyFromUsername("costantino_perciante"), "api add tag to test"); // } // // //@Test // public void removeTag()throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // instance.removeTag("test-after-tags-editing", instance.getApiKeyFromUsername("costantino_perciante"), "api add tag to test"); // // } // // //@Test // public void removeGroup()throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // instance.removeDatasetFromGroup("pending", "test-after-tags-editing", instance.getApiKeyFromUsername("costantino_perciante")); // // } // // //@Test // public void getGroups() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // instance.getParentGroups("abundance-level", instance.getApiKeyFromUsername("costantino_perciante")); // // } // // // @Test // public void createGroupsAndSetAsFather() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // //CkanGroup childGroup = instance.createGroup("test-group-child", "A child group", null); // //CkanGroup parentGroup = instance.createGroup("test-group-parent", "A parent group", null); // instance.setGroupParent("test-group-parent", "test-group-child"); // } // // //@Test // public void testBelongsToGroup() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // boolean checked = instance.isDatasetInGroup("assessment-unit", "test-after-updates-17-1654"); // logger.debug("Result is " + checked); // } // // // @Test // public void getDatasetsInGroup() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // List result = instance.getProductsInGroup("assessment-unit"); // for (CkanDataset ckanDataset : result) { // logger.debug("Dataset name is " + ckanDataset.getName()); // } // } // // // //@Test // public void getHigher(){ // logger.debug("Max is " + RolesCkanGroupOrOrg.getHigher(RolesCkanGroupOrOrg.ADMIN, RolesCkanGroupOrOrg.ADMIN)); // } // // //@Test // public void getUrlProduct() throws Exception{ // DataCatalogueImpl instance = factory.getUtilsPerScope(scope); // String datasetName = "test_from_andrea_rossi"; // String url = instance.getUrlFromDatasetIdOrName(datasetName); // logger.debug("url is " + url); // } // // //@Test // public void getCatalogueURLs() throws Exception{ // DataCatalogueImpl util = factory.getUtilsPerScope("/gcube/devNext/NextNext"); // logger.debug("MAP ACCESS URL TO CATALOGUE:" +util.getMapAccessURLToCatalogue()); // // } }