forked from D-Net/dnet-hadoop
refactoring and test
This commit is contained in:
parent
937ff6a7c7
commit
edcb17ca98
|
@ -56,7 +56,7 @@ public class PrepareResultCommunitySet {
|
||||||
log.info("production: {}", production);
|
log.info("production: {}", production);
|
||||||
|
|
||||||
final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production);
|
final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production);
|
||||||
//log.info("projectsMap: {}", new Gson().toJson(projectsMap));
|
// log.info("projectsMap: {}", new Gson().toJson(projectsMap));
|
||||||
|
|
||||||
SparkConf conf = new SparkConf();
|
SparkConf conf = new SparkConf();
|
||||||
|
|
||||||
|
|
|
@ -112,7 +112,7 @@ public class SparkResultToCommunityFromProject implements Serializable {
|
||||||
R ret = value._1();
|
R ret = value._1();
|
||||||
Optional<ResultProjectList> rcl = Optional.ofNullable(value._2());
|
Optional<ResultProjectList> rcl = Optional.ofNullable(value._2());
|
||||||
if (rcl.isPresent()) {
|
if (rcl.isPresent()) {
|
||||||
ArrayList<String> communitySet = rcl.get().getCommunityList();
|
// ArrayList<String> communitySet = rcl.get().getCommunityList();
|
||||||
List<String> contextList = ret
|
List<String> contextList = ret
|
||||||
.getContext()
|
.getContext()
|
||||||
.stream()
|
.stream()
|
||||||
|
@ -124,7 +124,7 @@ public class SparkResultToCommunityFromProject implements Serializable {
|
||||||
|
|
||||||
res.setId(ret.getId());
|
res.setId(ret.getId());
|
||||||
List<Context> propagatedContexts = new ArrayList<>();
|
List<Context> propagatedContexts = new ArrayList<>();
|
||||||
for (String cId : communitySet) {
|
for (String cId : rcl.get().getCommunityList()) {
|
||||||
if (!contextList.contains(cId)) {
|
if (!contextList.contains(cId)) {
|
||||||
Context newContext = new Context();
|
Context newContext = new Context();
|
||||||
newContext.setId(cId);
|
newContext.setId(cId);
|
||||||
|
@ -138,6 +138,20 @@ public class SparkResultToCommunityFromProject implements Serializable {
|
||||||
PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME,
|
PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME,
|
||||||
ModelConstants.DNET_PROVENANCE_ACTIONS)));
|
ModelConstants.DNET_PROVENANCE_ACTIONS)));
|
||||||
propagatedContexts.add(newContext);
|
propagatedContexts.add(newContext);
|
||||||
|
} else {
|
||||||
|
ret
|
||||||
|
.getContext()
|
||||||
|
.stream()
|
||||||
|
.filter(c -> c.getId().equals(cId))
|
||||||
|
.findFirst()
|
||||||
|
.get()
|
||||||
|
.getDataInfo()
|
||||||
|
.add(
|
||||||
|
getDataInfo(
|
||||||
|
PROPAGATION_DATA_INFO_TYPE,
|
||||||
|
PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID,
|
||||||
|
PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME,
|
||||||
|
ModelConstants.DNET_PROVENANCE_ACTIONS));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
res.setContext(propagatedContexts);
|
res.setContext(propagatedContexts);
|
||||||
|
|
|
@ -6,7 +6,6 @@ import static eu.dnetlib.dhp.bulktag.community.TaggingConstants.ZENODO_COMMUNITY
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
|
|
|
@ -8,7 +8,6 @@ import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.Context;
|
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
import org.apache.spark.api.java.JavaRDD;
|
import org.apache.spark.api.java.JavaRDD;
|
||||||
|
@ -27,6 +26,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.orcidtoresultfromsemrel.OrcidPropagationJobTest;
|
import eu.dnetlib.dhp.orcidtoresultfromsemrel.OrcidPropagationJobTest;
|
||||||
import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob;
|
import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Context;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Dataset;
|
import eu.dnetlib.dhp.schema.oaf.Dataset;
|
||||||
|
|
||||||
public class ResultToCommunityJobTest {
|
public class ResultToCommunityJobTest {
|
||||||
|
@ -98,23 +98,36 @@ public class ResultToCommunityJobTest {
|
||||||
* {"resultId":"50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1","communityList":["sdsn-gr"]}
|
* {"resultId":"50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1","communityList":["sdsn-gr"]}
|
||||||
* {"resultId":"50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1","communityList":["netherlands"]}
|
* {"resultId":"50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1","communityList":["netherlands"]}
|
||||||
*/
|
*/
|
||||||
List<Context> context = tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f"))
|
List<Context> context = tmp
|
||||||
.first().getContext();
|
.filter(r -> r.getId().equals("50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f"))
|
||||||
|
.first()
|
||||||
|
.getContext();
|
||||||
Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c)));
|
Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c)));
|
||||||
|
|
||||||
context = tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e"))
|
context = tmp
|
||||||
.first().getContext();
|
.filter(r -> r.getId().equals("50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e"))
|
||||||
|
.first()
|
||||||
|
.getContext();
|
||||||
Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c)));
|
Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c)));
|
||||||
|
|
||||||
Assertions.assertEquals(0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1")).count());
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1")).count());
|
||||||
|
|
||||||
Assertions.assertEquals(0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1")).count());
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1")).count());
|
||||||
|
|
||||||
Assertions.assertEquals(2, tmp.filter(r -> r.getContext().stream().anyMatch(c -> c.getId().equals("aurora"))).count());
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
2, tmp.filter(r -> r.getContext().stream().anyMatch(c -> c.getId().equals("aurora"))).count());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static boolean containsResultCommunityProject(Context c) {
|
private static boolean containsResultCommunityProject(Context c) {
|
||||||
return c.getDataInfo().stream().anyMatch(di -> di.getProvenanceaction().getClassid().equals("result:community:project"));
|
return c
|
||||||
|
.getDataInfo()
|
||||||
|
.stream()
|
||||||
|
.anyMatch(di -> di.getProvenanceaction().getClassid().equals("result:community:project"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue