D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Revert the changes for IgnoreUndefined management in tree evaluation #491

Merged
claudio.atzori merged 1 commits from fix_decision_tree into main 2024-10-11 10:33:42 +02:00
Conversation 1 Commits 1 Files Changed 3 +12 -31
3 changed files with 12 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/support/TreeNodeDef.java
View File

@ -48,7 +48,7 @@ public class TreeNodeDef implements Serializable {
// function for the evaluation of the node
public TreeNodeStats evaluate(Row doc1, Row doc2, Config conf) {
TreeNodeStats stats = new TreeNodeStats(ignoreUndefined);
TreeNodeStats stats = new TreeNodeStats();
// for each field in the node, it computes the
for (FieldConf fieldConf : fields) {

27
dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/support/TreeNodeStats.java
View File

@ -9,11 +9,8 @@ public class TreeNodeStats implements Serializable {
private Map<String, FieldStats> results; // this is an accumulator for the results of the node
private final boolean ignoreUndefined;
public TreeNodeStats(boolean ignoreUndefined) {
public TreeNodeStats() {
this.results = new HashMap<>();
this.ignoreUndefined = ignoreUndefined;
}
public Map<String, FieldStats> getResults() {
@ -25,10 +22,7 @@ public class TreeNodeStats implements Serializable {
}
public int fieldsCount() {
if (ignoreUndefined)
return this.results.size();
else
return this.results.size() - undefinedCount(); // do not count undefined
}
public int undefinedCount() {
@ -84,23 +78,12 @@ public class TreeNodeStats implements Serializable {
double min = 100.0; // random high value
for (FieldStats fs : this.results.values()) {
if (fs.getResult() < min) {
if (fs.getResult() == -1) {
if (fs.isCountIfUndefined()) {
min = 0.0;
} else {
min = -1;
}
} else {
if (fs.getResult() >= 0.0 || (fs.getResult() == -1 && fs.isCountIfUndefined()))
min = fs.getResult();
}
}
}
if (ignoreUndefined) {
return min == -1.0 ? 0.0 : min;
} else {
return min;
}
}
// if at least one is true, return 1.0
public double or() {
@ -108,12 +91,8 @@ public class TreeNodeStats implements Serializable {
if (fieldStats.getResult() >= fieldStats.getThreshold())
return 1.0;
}
if (!ignoreUndefined && undefinedCount() > 0) {
return -1.0;
} else {
return 0.0;
}
}
// if at least one is false, return 0.0
public double and() {
@ -121,7 +100,7 @@ public class TreeNodeStats implements Serializable {
if (fieldStats.getResult() == -1) {
if (fieldStats.isCountIfUndefined())
return ignoreUndefined ? 0.0 : -1.0;
return 0.0;
} else {
if (fieldStats.getResult() < fieldStats.getThreshold())
return 0.0;

8
dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/support/TreeProcessor.java
View File

@ -44,10 +44,12 @@ public class TreeProcessor {
TreeNodeStats stats = currentNode.evaluate(doc1, doc2, config);
treeStats.addNodeStats(nextNodeName, stats);
double finalScore = stats.getFinalScore(currentNode.getAggregation());
if (finalScore == -1.0)
// if ignoreUndefined=false the miss is considered as undefined
if (!currentNode.isIgnoreUndefined() && stats.undefinedCount() > 0) {
nextNodeName = currentNode.getUndefined();
else if (finalScore >= currentNode.getThreshold()) {
}
// if ignoreUndefined=true the miss is ignored and the score computed anyway
else if (stats.getFinalScore(currentNode.getAggregation()) >= currentNode.getThreshold()) {
nextNodeName = currentNode.getPositive();
} else {
nextNodeName = currentNode.getNegative();