D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

[ENRICHMENT][BETA] Use of community API in enrichment process AND addition to tagging result for communities through projects #359

Merged
claudio.atzori merged 22 commits from propagationapi into beta 2023-11-30 14:20:34 +01:00
Conversation 0 Commits 22 Files Changed 103 +13 -16
4 changed files with 13 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 8eb70e6657 - Show all commits

4
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java
View File

@ -79,8 +79,8 @@ public class PrepareAffiliationRelationsTest {
.getPath(); .getPath();
String pubmedAffiliationRelationsPath = getClass() String pubmedAffiliationRelationsPath = getClass()
.getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json")
.getPath(); .getPath();
String outputPath = workingDir.toString() + "/actionSet"; String outputPath = workingDir.toString() + "/actionSet";

21
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java
View File

@ -15,8 +15,6 @@ import org.jetbrains.annotations.NotNull;
*/ */
public class QueryCommunityAPI { public class QueryCommunityAPI {
claudio.atzori marked this conversation as resolved Outdated
claudio.atzori commented 2023-11-03 15:53:44 +01:00
Outdated
Review
Copy Link

It is not a good idea to hardcode the service base urls. Please consider to factor it out as an input parameter: having the possibility to define the API base URL fits well with the definition of a test case where the API server application is mocked locally.

It is not a good idea to hardcode the service base urls. Please consider to factor it out as an input parameter: having the possibility to define the API base URL fits well with the definition of a test case where the API server application is mocked locally.
miriam.baglioni commented 2023-11-27 15:16:08 +01:00
Outdated
Review
Copy Link

I have changed this. Now we have a parameter baseURL where to specify the URL for production or beta.
It is expected to be "https://services.openaire.eu/openaire/communities/" and the equivalent one for beta "https://beta.services.openaire.eu/openaire/communities/"

I have changed this. Now we have a parameter baseURL where to specify the URL for production or beta. It is expected to be "https://services.openaire.eu/openaire/communities/" and the equivalent one for beta "https://beta.services.openaire.eu/openaire/communities/"
claudio.atzori commented 2023-11-28 11:58:51 +01:00
Outdated
Review
Copy Link

Thanks!

Thanks!
private static String get(String geturl) throws IOException { private static String get(String geturl) throws IOException {
URL url = new URL(geturl); URL url = new URL(geturl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); HttpURLConnection conn = (HttpURLConnection) url.openConnection();
@ -34,32 +32,31 @@ public class QueryCommunityAPI {
public static String communities(String baseURL) throws IOException { public static String communities(String baseURL) throws IOException {
return get(baseURL + "communities"); return get(baseURL + "communities");
} }
public static String community(String id, String baseURL ) throws IOException { public static String community(String id, String baseURL) throws IOException {
return get(baseURL + id); return get(baseURL + id);
} }
public static String communityDatasource(String id, String baseURL ) throws IOException { public static String communityDatasource(String id, String baseURL) throws IOException {
return get(baseURL + id + "/contentproviders");
return get(baseURL + id + "/contentproviders");
} }
public static String communityPropagationOrganization(String id, String baseURL ) throws IOException { public static String communityPropagationOrganization(String id, String baseURL) throws IOException {
return get(baseURL + id + "/propagationOrganizations"); return get(baseURL + id + "/propagationOrganizations");
} }
public static String communityProjects(String id, String page, String size, String baseURL ) throws IOException { public static String communityProjects(String id, String page, String size, String baseURL) throws IOException {
return get(baseURL + id + "/projects/" + page + "/" + size); return get(baseURL + id + "/projects/" + page + "/" + size);
} }

2
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java
View File

@ -52,7 +52,7 @@ public class PrepareResultCommunitySet {
final String outputPath = parser.get("outputPath"); final String outputPath = parser.get("outputPath");
log.info("outputPath: {}", outputPath); log.info("outputPath: {}", outputPath);
final String baseURL = parser.get("baseURL"); final String baseURL = parser.get("baseURL");
log.info("baseUEL: {}", baseURL); log.info("baseUEL: {}", baseURL);
final CommunityEntityMap projectsMap = Utils.getCommunityProjects(baseURL); final CommunityEntityMap projectsMap = Utils.getCommunityProjects(baseURL);

2
dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java
View File

@ -1576,7 +1576,7 @@ public class BulkTagJobTest {
"-outputPath", workingDir.toString() + "/", "-outputPath", workingDir.toString() + "/",
// "-baseURL", "https://services.openaire.eu/openaire/community/", // "-baseURL", "https://services.openaire.eu/openaire/community/",
"-pathMap", pathMap, "-pathMap", pathMap,
"-taggingConf", taggingConf "-taggingConf", taggingConf
}); });
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());