apply commits from master

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/base/BaseCollectorPlugin.java

@@ -45,15 +45,22 @@ public class BaseCollectorPlugin implements CollectorPlugin {
 
 	@Override
 	public Stream<String> collect(final ApiDescriptor api, final AggregatorReport report) throws CollectorException {
-		// get path to file
+		// the path of the dump file on HDFS
+		// http://oai.base-search.net/initial_load/base_oaipmh_dump-current.tar
+		// it could be downloaded from iis-cdh5-test-gw.ocean.icm.edu.pl and then copied on HDFS
 		final Path filePath = Optional
 			.ofNullable(api.getBaseUrl())
 			.map(Path::new)
 			.orElseThrow(() -> new CollectorException("missing baseUrl"));
 
+		// get the parameters for the connection to the OpenAIRE database.
+		// the database is used to obtain the list of the datasources that the plugin will collect
 		final String dbUrl = api.getParams().get("dbUrl");
 		final String dbUser = api.getParams().get("dbUser");
 		final String dbPassword = api.getParams().get("dbPassword");
+
+		// the types(comma separated, empty value for all) that the plugin will collect,
+		// the types should be expressed in the format of the normalized types of BASE (for example 1,121,...)
 		final String acceptedNormTypesString = api.getParams().get("acceptedNormTypes");
 
 		log.info("baseUrl: {}", filePath);

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/plugin/base/sql/base.sql

@@ -108,7 +108,7 @@ INSERT INTO dsm_apiparams(
 	'api_________::openaire____::base_search::dump@@acceptedNormTypes',
 	'api_________::openaire____::base_search::dump',
 	'acceptedNormTypes',
-	''
+	'1,11,111,121,13,14,15,18,181,182,183,1A,6,7'
 );
 
 COMMIT;

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/plugin/base/sql/opendoar-accepted.sql

@@ -2,6 +2,8 @@ select s.id as id
 from dsm_services s 
 where collectedfrom = 'openaire____::opendoar' 
 and jurisdiction = 'Institutional'
-and s.id not in (
-	select service from dsm_api where coalesce(compatibility_override, compatibility) like '%openaire%' or last_collection_total > 0
-);
+and s.id in (
+	select service from dsm_api where coalesce(compatibility_override, compatibility) = 'driver' or coalesce(compatibility_override, compatibility) = 'UNKNOWN'
+) and s.id not in (
+	select service from dsm_api where coalesce(compatibility_override, compatibility) like '%openaire%'
+);