minor changes
This commit is contained in:
parent
6a27fb14a8
commit
cc86591fad
|
@ -49,7 +49,7 @@ public class Deduper implements Serializable {
|
||||||
|
|
||||||
System.out.println("Number of relations = " + relationRDD.distinct().count());
|
System.out.println("Number of relations = " + relationRDD.distinct().count());
|
||||||
|
|
||||||
final RDD<Edge<String>> edgeRdd = relationRDD.map(it -> new Edge<>(Utility.getHashcode(it._1()),Utility.getHashcode(it._2()), "equalTo")).rdd();
|
final RDD<Edge<String>> edgeRdd = relationRDD.map(it -> new Edge<>(Utility.getHashcode(it._1()),Utility.getHashcode(it._2()), "isSimilarTo")).rdd();
|
||||||
|
|
||||||
accumulators.forEach((name, acc) -> log.info(name + " -> " + acc.value()));
|
accumulators.forEach((name, acc) -> log.info(name + " -> " + acc.value()));
|
||||||
|
|
||||||
|
|
|
@ -87,7 +87,7 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"threshold": 0.7,
|
"threshold": 0.1,
|
||||||
"aggregation": "W_MEAN",
|
"aggregation": "W_MEAN",
|
||||||
"positive": "layer4",
|
"positive": "layer4",
|
||||||
"negative": "NO_MATCH",
|
"negative": "NO_MATCH",
|
||||||
|
@ -106,7 +106,7 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"threshold": 0.9,
|
"threshold": 0.7,
|
||||||
"aggregation": "AVG",
|
"aggregation": "AVG",
|
||||||
"positive": "layer5",
|
"positive": "layer5",
|
||||||
"negative": "NO_MATCH",
|
"negative": "NO_MATCH",
|
||||||
|
@ -129,7 +129,9 @@
|
||||||
"comparator": "jaroWinklerNormalizedName",
|
"comparator": "jaroWinklerNormalizedName",
|
||||||
"weight": 0.1,
|
"weight": 0.1,
|
||||||
"countIfUndefined": "false",
|
"countIfUndefined": "false",
|
||||||
"params": {}
|
"params": {
|
||||||
|
"windowSize": 4
|
||||||
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"threshold": 0.9,
|
"threshold": 0.9,
|
||||||
|
|
|
@ -66,13 +66,12 @@
|
||||||
"weight": 1.0,
|
"weight": 1.0,
|
||||||
"countIfUndefined": "false",
|
"countIfUndefined": "false",
|
||||||
"params": {
|
"params": {
|
||||||
"threshold": "0.5",
|
|
||||||
"jpath_value": "$.value",
|
"jpath_value": "$.value",
|
||||||
"jpath_classid": "$.qualifier.classid"
|
"jpath_classid": "$.qualifier.classid"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"threshold": 1.0,
|
"threshold": 0.5,
|
||||||
"aggregation": "MAX",
|
"aggregation": "MAX",
|
||||||
"positive": "MATCH",
|
"positive": "MATCH",
|
||||||
"negative": "layer2",
|
"negative": "layer2",
|
||||||
|
|
Loading…
Reference in New Issue