lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

translation map moved in json configuration, support for synonyms added in the configuration, now the configuration is argument of conditions, distancealgos and clusteringfunctions

This commit is contained in:
miconis 2019-10-08 14:53:52 +02:00
parent 07355d2811
commit 26b383fea2
48 changed files with 304 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/AbstractClusteringFunction.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.clustering;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.model.Field;
import org.apache.commons.lang.StringUtils;
@ -18,15 +19,15 @@ public abstract class AbstractClusteringFunction extends AbstractPaceFunctions i
this.params = params;
}
protected abstract Collection<String> doApply(String s);
protected abstract Collection<String> doApply(Config conf, String s);
@Override
public Collection<String> apply(List<Field> fields) {
public Collection<String> apply(Config conf, List<Field> fields) {
return fields.stream().filter(f -> !f.isEmpty())
.map(Field::stringValue)
.map(this::normalize)
.map(s -> filterAllStopWords(s))
.map(this::doApply)
.map(s -> doApply(conf, s))
.map(c -> filterBlacklisted(c, ngramBlacklist))
.flatMap(c -> c.stream())
.filter(StringUtils::isNotBlank)

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/Acronyms.java
View File

@ -6,6 +6,7 @@ import java.util.Set;
import java.util.StringTokenizer;
import com.google.common.collect.Sets;
import eu.dnetlib.pace.config.Config;
@ClusteringClass("acronyms")
public class Acronyms extends AbstractClusteringFunction {
@ -15,7 +16,7 @@ public class Acronyms extends AbstractClusteringFunction {
}
@Override
protected Collection<String> doApply(String s) {
protected Collection<String> doApply(Config conf, String s) {
return extractAcronyms(s, param("max"), param("minLen"), param("maxLen"));
}

8
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/ClusteringCombiner.java
View File

@ -13,15 +13,15 @@ import eu.dnetlib.pace.model.Field;
public class ClusteringCombiner {
public static Collection<String> combine(final Document a, final Config conf) {
return new ClusteringCombiner().doCombine(a, conf.clusterings());
return new ClusteringCombiner().doCombine(a, conf);
}
private Collection<String> doCombine(final Document a, final List<ClusteringDef> defs) {
private Collection<String> doCombine(final Document a, final Config conf) {
final Collection<String> res = Sets.newLinkedHashSet();
for (final ClusteringDef cd : defs) {
for (final ClusteringDef cd : conf.clusterings()) {
for (final String fieldName : cd.getFields()) {
final Field values = a.values(fieldName);
res.addAll(cd.clusteringFunction().apply((List<Field>) values));
res.addAll(cd.clusteringFunction().apply(conf, (List<Field>) values));
}
}
return res;

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/ClusteringFunction.java
View File

@ -4,11 +4,12 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.model.Field;
public interface ClusteringFunction {
public Collection<String> apply(List<Field> fields);
public Collection<String> apply(Config config, List<Field> fields);
public Map<String, Integer> getParams();

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/ImmutableFieldValue.java
View File

@ -5,6 +5,7 @@ import java.util.List;
import java.util.Map;
import com.google.common.collect.Lists;
import eu.dnetlib.pace.config.Config;
@ClusteringClass("immutablefieldvalue")
public class ImmutableFieldValue extends AbstractClusteringFunction {
@ -14,7 +15,7 @@ public class ImmutableFieldValue extends AbstractClusteringFunction {
}
@Override
protected Collection<String> doApply(final String s) {
protected Collection<String> doApply(final Config conf, final String s) {
final List<String> res = Lists.newArrayList();
res.add(s);

9
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/KeywordsClustering.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.clustering;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.model.Field;
import org.apache.commons.lang.StringUtils;
@ -15,10 +16,10 @@ public class KeywordsClustering extends AbstractClusteringFunction {
}
@Override
protected Collection<String> doApply(String s) {
protected Collection<String> doApply(final Config conf, String s) {
//takes city codes and keywords codes without duplicates
Set<String> keywords = getKeywords(s, params.getOrDefault("windowSize", 4));
Set<String> keywords = getKeywords(s, conf.translationMap(), params.getOrDefault("windowSize", 4));
Set<String> cities = getCities(s, params.getOrDefault("windowSize", 4));
//list of combination to return as result
@ -37,13 +38,13 @@ public class KeywordsClustering extends AbstractClusteringFunction {
}
@Override
public Collection<String> apply(List<Field> fields) {
public Collection<String> apply(final Config conf, List<Field> fields) {
return fields.stream().filter(f -> !f.isEmpty())
.map(Field::stringValue)
.map(this::cleanup) //TODO can I add this to the AbstractClusteringFunction without overriding the method here?
.map(this::normalize)
.map(s -> filterAllStopWords(s))
.map(this::doApply)
.map(s -> doApply(conf, s))
.map(c -> filterBlacklisted(c, ngramBlacklist))
.flatMap(c -> c.stream())
.filter(StringUtils::isNotBlank)

7
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/LowercaseClustering.java
View File

@ -6,6 +6,7 @@ import java.util.Map;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.model.Field;
import org.apache.commons.lang.StringUtils;
@ -17,16 +18,16 @@ public class LowercaseClustering extends AbstractClusteringFunction {
}
@Override
public Collection<String> apply(List<Field> fields) {
public Collection<String> apply(Config conf, List<Field> fields) {
Collection<String> c = Sets.newLinkedHashSet();
for(Field f : fields) {
c.addAll(doApply(f.stringValue()));
c.addAll(doApply(conf, f.stringValue()));
}
return c;
}
@Override
protected Collection<String> doApply(final String s) {
protected Collection<String> doApply(final Config conf, final String s) {
if(StringUtils.isBlank(s)) {
return Lists.newArrayList();
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/NgramPairs.java
View File

@ -6,6 +6,7 @@ import java.util.List;
import java.util.Map;
import com.google.common.collect.Lists;
import eu.dnetlib.pace.config.Config;
@ClusteringClass("ngrampairs")
public class NgramPairs extends Ngrams {
@ -15,7 +16,7 @@ public class NgramPairs extends Ngrams {
}
@Override
protected Collection<String> doApply(String s) {
protected Collection<String> doApply(Config conf, String s) {
return ngramPairs(Lists.newArrayList(getNgrams(s, param("ngramLen"), param("max") * 2, 1, 2)), param("max"));
}

4
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/Ngrams.java
View File

@ -1,5 +1,7 @@
package eu.dnetlib.pace.clustering;
import eu.dnetlib.pace.config.Config;
import java.util.*;
@ClusteringClass("ngrams")
@ -10,7 +12,7 @@ public class Ngrams extends AbstractClusteringFunction {
}
@Override
protected Collection<String> doApply(String s) {
protected Collection<String> doApply(Config conf, String s) {
return getNgrams(s, param("ngramLen"), param("max"), param("maxPerToken"), param("minNgramLen"));
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/PersonClustering.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.pace.clustering;
import com.google.common.collect.Sets;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.Person;
import org.apache.commons.lang.StringUtils;
@ -23,7 +24,7 @@ public class PersonClustering extends AbstractPaceFunctions implements Clusterin
}
@Override
public Collection<String> apply(final List<Field> fields) {
public Collection<String> apply(final Config conf, final List<Field> fields) {
final Set<String> hashes = Sets.newHashSet();
for (final Field f : fields) {

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/PersonHash.java
View File

@ -6,6 +6,7 @@ import java.util.Map;
import com.google.common.collect.Lists;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.model.Person;
@ClusteringClass("personhash")
@ -18,7 +19,7 @@ public class PersonHash extends AbstractClusteringFunction {
}
@Override
protected Collection<String> doApply(final String s) {
protected Collection<String> doApply(final Config conf, final String s) {
final List<String> res = Lists.newArrayList();
final boolean aggressive = (Boolean) (getParams().containsKey("aggressive") ? getParams().get("aggressive") : DEFAULT_AGGRESSIVE);

4
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/RandomClusteringFunction.java
View File

@ -1,5 +1,7 @@
package eu.dnetlib.pace.clustering;
import eu.dnetlib.pace.config.Config;
import java.util.Collection;
import java.util.Map;
@ -10,7 +12,7 @@ public class RandomClusteringFunction extends AbstractClusteringFunction {
}
@Override
protected Collection<String> doApply(String s) {
protected Collection<String> doApply(final Config conf, String s) {
// TODO Auto-generated method stub
return null;
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/SortedNgramPairs.java
View File

@ -5,6 +5,7 @@ import java.util.*;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import eu.dnetlib.pace.config.Config;
@ClusteringClass("sortedngrampairs")
public class SortedNgramPairs extends NgramPairs {
@ -14,7 +15,7 @@ public class SortedNgramPairs extends NgramPairs {
}
@Override
protected Collection<String> doApply(String s) {
protected Collection<String> doApply(Config conf, String s) {
final List<String> tokens = Lists.newArrayList(Splitter.on(" ").omitEmptyStrings().trimResults().split(s));

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/SpaceTrimmingFieldValue.java
View File

@ -4,6 +4,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import eu.dnetlib.pace.config.Config;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang.StringUtils;
@ -17,7 +18,7 @@ public class SpaceTrimmingFieldValue extends AbstractClusteringFunction {
}
@Override
protected Collection<String> doApply(final String s) {
protected Collection<String> doApply(final Config conf, final String s) {
final List<String> res = Lists.newArrayList();
res.add(StringUtils.isBlank(s) ? RandomStringUtils.random(getParams().get("randomLength")) : s.toLowerCase().replaceAll("\\s+", ""));

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/SuffixPrefix.java
View File

@ -5,6 +5,7 @@ import java.util.Map;
import java.util.Set;
import com.google.common.collect.Sets;
import eu.dnetlib.pace.config.Config;
@ClusteringClass("suffixprefix")
public class SuffixPrefix extends AbstractClusteringFunction {
@ -14,7 +15,7 @@ public class SuffixPrefix extends AbstractClusteringFunction {
}
@Override
protected Collection<String> doApply(String s) {
protected Collection<String> doApply(Config conf, String s) {
return suffixPrefix(s, param("len"), param("max"));
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/clustering/UrlClustering.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.clustering;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.model.Field;
import java.net.MalformedURLException;
@ -21,7 +22,7 @@ public class UrlClustering extends AbstractPaceFunctions implements ClusteringFu
}
@Override
public Collection<String> apply(List<Field> fields) {
public Collection<String> apply(final Config conf, List<Field> fields) {
try {
return fields.stream()
.filter(f -> !f.isEmpty())

15
dnet-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java
View File

@ -13,6 +13,8 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.io.StringWriter;
import java.text.Normalizer;
import java.util.*;
import java.util.regex.Matcher;
@ -327,12 +329,17 @@ public abstract class AbstractPaceFunctions {
return codes;
}
public Set<String> getKeywords(String s1, int windowSize) {
return getKeywords(s1, translationMap, windowSize);
}
public Set<String> getCities(String s1, int windowSize) {
return getKeywords(s1, cityMap, windowSize);
}
public static <T> String readFromClasspath(final String filename, final Class<T> clazz) {
final StringWriter sw = new StringWriter();
try {
IOUtils.copy(clazz.getResourceAsStream(filename), sw);
return sw.toString();
} catch (final IOException e) {
throw new RuntimeException("cannot load resource from classpath: " + filename);
}
}
}

9
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/AbstractCondition.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.pace.condition;
import java.util.List;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEval;
import eu.dnetlib.pace.distance.eval.ConditionEvalMap;
import eu.dnetlib.pace.model.Document;
@ -25,10 +26,10 @@ public abstract class AbstractCondition extends AbstractPaceFunctions implements
this.fields = fields;
}
protected abstract ConditionEval verify(FieldDef fd, Field a, Field b);
protected abstract ConditionEval verify(FieldDef fd, Field a, Field b, Config conf);
@Override
public ConditionEvalMap verify(final Document a, final Document b) {
public ConditionEvalMap verify(final Document a, final Document b, final Config conf) {
final ConditionEvalMap res = new ConditionEvalMap();
for (final FieldDef fd : getFields()) {
@ -36,12 +37,12 @@ public abstract class AbstractCondition extends AbstractPaceFunctions implements
final Field vb = b.values(fd.getName());
if (fd.isIgnoreMissing()) {
res.put(fd.getName(), verify(fd, va, vb));
res.put(fd.getName(), verify(fd, va, vb, conf));
} else {
if (va.isEmpty() || vb.isEmpty()) {
res.put(fd.getName(), new ConditionEval(cond, va, vb, -1));
} else {
res.put(fd.getName(), verify(fd, va, vb));
res.put(fd.getName(), verify(fd, va, vb, conf));
}
}
}

4
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/AlwaysTrueCondition.java
View File

@ -1,6 +1,8 @@
package eu.dnetlib.pace.condition;
import java.util.List;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEval;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldDef;
@ -18,7 +20,7 @@ public class AlwaysTrueCondition extends AbstractCondition {
}
@Override
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b) {
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b, final Config conf) {
return new ConditionEval(cond, a, b, 1);
}

4
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/ConditionAlgo.java
View File

@ -1,6 +1,8 @@
package eu.dnetlib.pace.condition;
import java.util.List;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEvalMap;
import eu.dnetlib.pace.model.Document;
import eu.dnetlib.pace.model.FieldDef;
@ -22,6 +24,6 @@ public interface ConditionAlgo {
* @return 0 when condition cannot be verified (ignoremissing = true). Positive int when the condition is verified. Negative int when
* the condition is not verified.
*/
public abstract ConditionEvalMap verify(Document a, Document b);
public abstract ConditionEvalMap verify(Document a, Document b, Config conf);
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/ExactMatch.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.pace.condition;
import java.util.List;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEval;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldDef;
@ -20,7 +21,7 @@ public class ExactMatch extends AbstractCondition {
}
@Override
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b) {
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b, final Config conf) {
final String fa = getValue(a);
final String fb = getValue(b);

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/ExactMatchIgnoreCase.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.pace.condition;
import java.util.List;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEval;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldDef;
@ -20,7 +21,7 @@ public class ExactMatchIgnoreCase extends AbstractCondition {
}
@Override
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b) {
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b, final Config conf) {
final String fa = getValue(a);
final String fb = getValue(b);

4
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/MustBeDifferent.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.pace.condition;
import java.util.List;
import com.google.common.collect.Iterables;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEval;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldDef;
@ -30,7 +31,8 @@ public class MustBeDifferent extends AbstractCondition {
* @see eu.dnetlib.pace.condition.AbstractCondition#verify(eu.dnetlib.pace.model.FieldDef, java.util.List, java.util.List)
*/
@Override
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b) {
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b, final Config conf)
{
final String fa = getValue(a);
final String fb = getValue(b);

4
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/PidMatch.java
View File

@ -6,6 +6,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import com.google.common.collect.Sets;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEval;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldDef;
@ -29,7 +30,8 @@ public class PidMatch extends AbstractCondition {
}
@Override
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b) {
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b, final Config conf
) {
final List<String> sa = ((FieldList) a).stringList();
final List<String> sb = ((FieldList) b).stringList();

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/SizeMatch.java
View File

@ -4,6 +4,7 @@ import java.util.List;
import com.google.common.collect.Iterables;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEval;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldDef;
@ -32,7 +33,7 @@ public class SizeMatch extends AbstractCondition {
* @see eu.dnetlib.pace.condition.AbstractCondition#verify(eu.dnetlib.pace.model.FieldDef, java.util.List, java.util.List)
*/
@Override
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b) {
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b, final Config conf) {
// if (a.isEmpty() & b.isEmpty()) return 1;
//

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/TitleVersionMatch.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.pace.condition;
import java.util.List;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEval;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldDef;
@ -20,7 +21,7 @@ public class TitleVersionMatch extends AbstractCondition {
}
@Override
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b) {
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b, final Config conf) {
final String valueA = getFirstValue(a);
final String valueB = getFirstValue(b);

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/condition/YearMatch.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.pace.condition;
import java.time.Year;
import java.util.List;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.eval.ConditionEval;
import org.apache.commons.lang.StringUtils;
@ -34,7 +35,7 @@ public class YearMatch extends AbstractCondition {
// }
@Override
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b) {
protected ConditionEval verify(final FieldDef fd, final Field a, final Field b, final Config conf) {
final String valueA = getNumbers(getFirstValue(a));
final String valueB = getNumbers(getFirstValue(b));

2
dnet-pace-core/src/main/java/eu/dnetlib/pace/config/Config.java
View File

@ -56,4 +56,6 @@ public interface Config {
*/
public Map<String, List<String>> blacklists();
public Map<String, String> translationMap();
}

6
dnet-pace-core/src/main/java/eu/dnetlib/pace/config/DedupConfig.java
View File

@ -55,6 +55,7 @@ public class DedupConfig implements Config, Serializable {
try {
config = new ObjectMapper().readValue(json, DedupConfig.class);
config.getPace().initModel();
config.getPace().initTranslationMap();
return config;
} catch (IOException e) {
throw new PaceException("Error in parsing configuration json", e);
@ -144,4 +145,9 @@ public class DedupConfig implements Config, Serializable {
return getPace().getBlacklists();
}
@Override
public Map<String, String> translationMap() {
return getPace().translationMap();
}
}

25
dnet-pace-core/src/main/java/eu/dnetlib/pace/config/PaceConfig.java
View File

@ -22,6 +22,10 @@ public class PaceConfig implements Serializable {
private List<CondDef> conditions;
private List<ClusteringDef> clustering;
private Map<String, List<String>> blacklists;
private Map<String, List<String>> synonyms;
@JsonIgnore
private Map<String, String> translationMap;
@JsonIgnore
private Map<String, FieldDef> modelMap;
@ -38,6 +42,19 @@ public class PaceConfig implements Serializable {
}
}
public void initTranslationMap(){
translationMap = Maps.newHashMap();
for (String key : synonyms.keySet()) {
for (String term : synonyms.get(key)){
translationMap.put(term.toLowerCase(), key);
}
}
}
public Map<String, String> translationMap(){
return translationMap;
}
public List<FieldDef> getModel() {
return model;
}
@ -88,6 +105,14 @@ public class PaceConfig implements Serializable {
this.blacklists = blacklists;
}
public Map<String, List<String>> getSynonyms() {
return synonyms;
}
public void setSynonyms(Map<String, List<String>> synonyms) {
this.synonyms = synonyms;
}
public Map<String, FieldDef> getModelMap() {
return modelMap;
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/DistanceAlgo.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.pace.distance;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.model.Field;
import java.util.Map;
@ -10,7 +11,7 @@ import java.util.Map;
*/
public interface DistanceAlgo {
public abstract double distance(Field a, Field b);
public abstract double distance(Field a, Field b, Config conf);
public double getWeight();

4
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/DistanceScorer.java
View File

@ -49,7 +49,7 @@ public class DistanceScorer {
final ConditionEvalMap res = new ConditionEvalMap();
for (final ConditionAlgo cd : conditions) {
final ConditionEvalMap map = cd.verify(a, b);
final ConditionEvalMap map = cd.verify(a, b, config);
res.mergeFrom(map);
// commented out shortcuts
@ -82,7 +82,7 @@ public class DistanceScorer {
}
} else {
if (va.getType().equals(vb.getType())) {
de.setDistance(w * fd.distanceAlgo().distance(va, vb));
de.setDistance(w * fd.distanceAlgo().distance(va, vb, config));
} else {
throw new PaceException(String.format("Types are different: %s:%s - %s:%s", va, va.getType(), vb, vb.getType()));
}

13
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/SecondStringDistanceAlgo.java
View File

@ -7,6 +7,7 @@ import java.util.Map;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.config.Type;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.model.FieldList;
@ -69,7 +70,7 @@ public abstract class SecondStringDistanceAlgo extends AbstractPaceFunctions imp
* the b
* @return the double
*/
public double distance(final String a, final String b) {
public double distance(final String a, final String b, final Config conf) {
double score = ssalgo.score(a, b);
return normalize(score);
}
@ -83,8 +84,8 @@ public abstract class SecondStringDistanceAlgo extends AbstractPaceFunctions imp
* the b
* @return the double
*/
protected double distance(final List<String> a, final List<String> b) {
return distance(concat(a), concat(b));
protected double distance(final List<String> a, final List<String> b, final Config conf) {
return distance(concat(a), concat(b), conf);
}
/*
@ -93,9 +94,9 @@ public abstract class SecondStringDistanceAlgo extends AbstractPaceFunctions imp
* @see eu.dnetlib.pace.distance.DistanceAlgo#distance(eu.dnetlib.pace.model.Field, eu.dnetlib.pace.model.Field)
*/
@Override
public double distance(final Field a, final Field b) {
if (a.getType().equals(Type.String) && b.getType().equals(Type.String)) return distance(a.stringValue(), b.stringValue());
if (a.getType().equals(Type.List) && b.getType().equals(Type.List)) return distance(toList(a), toList(b));
public double distance(final Field a, final Field b, final Config conf) {
if (a.getType().equals(Type.String) && b.getType().equals(Type.String)) return distance(a.stringValue(), b.stringValue(), conf);
if (a.getType().equals(Type.List) && b.getType().equals(Type.List)) return distance(toList(a), toList(b), conf);
throw new IllegalArgumentException("invalid types\n- A: " + a.toString() + "\n- B: " + b.toString());
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/AlwaysMatch.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.distance.algo;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
@ -22,7 +23,7 @@ public class AlwaysMatch extends SecondStringDistanceAlgo {
}
@Override
public double distance(final String a, final String b) {
public double distance(final String a, final String b, final Config conf) {
return 1.0;
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/ExactMatch.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.distance.algo;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
@ -22,7 +23,7 @@ public class ExactMatch extends SecondStringDistanceAlgo {
}
@Override
public double distance(final String a, final String b) {
public double distance(final String a, final String b, final Config conf) {
return a.equals(b) ? 1.0 : 0;
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/JaroWinkler.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.distance.algo;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
@ -24,7 +25,7 @@ public class JaroWinkler extends SecondStringDistanceAlgo {
}
@Override
public double distance(String a, String b) {
public double distance(String a, String b, final Config conf) {
String ca = cleanup(a);
String cb = cleanup(b);

7
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/JaroWinklerNormalizedName.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.pace.distance.algo;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
@ -27,7 +28,7 @@ public class JaroWinklerNormalizedName extends SecondStringDistanceAlgo {
}
@Override
public double distance(String a, String b) {
public double distance(String a, String b, final Config conf) {
String ca = cleanup(a);
String cb = cleanup(b);
@ -37,8 +38,8 @@ public class JaroWinklerNormalizedName extends SecondStringDistanceAlgo {
ca = filterAllStopWords(ca);
cb = filterAllStopWords(cb);
Set<String> keywords1 = getKeywords(ca, params.getOrDefault("windowSize", 4).intValue());
Set<String> keywords2 = getKeywords(cb, params.getOrDefault("windowSize", 4).intValue());
Set<String> keywords1 = getKeywords(ca, conf.translationMap(), params.getOrDefault("windowSize", 4).intValue());
Set<String> keywords2 = getKeywords(cb, conf.translationMap(), params.getOrDefault("windowSize", 4).intValue());
Set<String> cities1 = getCities(ca, params.getOrDefault("windowSize", 4).intValue());
Set<String> cities2 = getCities(cb, params.getOrDefault("windowSize", 4).intValue());

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/JaroWinklerTitle.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.distance.algo;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
@ -23,7 +24,7 @@ public class JaroWinklerTitle extends SecondStringDistanceAlgo {
}
@Override
public double distance(String a, String b) {
public double distance(String a, String b, final Config conf) {
String ca = cleanup(a);
String cb = cleanup(b);

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/Level2JaroWinklerTitle.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.distance.algo;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
@ -22,7 +23,7 @@ public class Level2JaroWinklerTitle extends SecondStringDistanceAlgo {
}
@Override
public double distance(final String a, final String b) {
public double distance(final String a, final String b, final Config conf) {
final String ca = cleanup(a);
final String cb = cleanup(b);

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/LevensteinTitle.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.distance.algo;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.DistanceScorer;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
@ -27,7 +28,7 @@ public class LevensteinTitle extends SecondStringDistanceAlgo {
}
@Override
public double distance(final String a, final String b) {
public double distance(final String a, final String b, final Config conf) {
final String ca = cleanup(a);
final String cb = cleanup(b);

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/LevensteinTitleIgnoreVersion.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.distance.algo;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
@ -25,7 +26,7 @@ public class LevensteinTitleIgnoreVersion extends SecondStringDistanceAlgo {
}
@Override
public double distance(final String a, final String b) {
public double distance(final String a, final String b, final Config conf) {
String ca = cleanup(a);
String cb = cleanup(b);

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/MustBeDifferent.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.distance.algo;
import com.wcohen.ss.AbstractStringDistance;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
@ -22,7 +23,7 @@ public class MustBeDifferent extends SecondStringDistanceAlgo {
}
@Override
public double distance(final String a, final String b) {
public double distance(final String a, final String b, final Config conf) {
return !a.equals(b) ? 1.0 : 0;
}

3
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/NullDistanceAlgo.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.pace.distance.algo;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceAlgo;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.model.Field;
@ -17,7 +18,7 @@ public class NullDistanceAlgo implements DistanceAlgo {
}
@Override
public double distance(Field a, Field b) {
public double distance(Field a, Field b, final Config conf) {
return 0.0;
}

5
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/SubStringLevenstein.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.pace.distance.algo;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.distance.SecondStringDistanceAlgo;
import org.apache.commons.lang.StringUtils;
@ -69,9 +70,9 @@ public class SubStringLevenstein extends SecondStringDistanceAlgo {
* @see eu.dnetlib.pace.distance.SecondStringDistanceAlgo#distance(eu.dnetlib.pace.model.Field, eu.dnetlib.pace.model.Field)
*/
@Override
public double distance(final Field a, final Field b) {
public double distance(final Field a, final Field b, final Config conf) {
if (a.getType().equals(Type.String) && b.getType().equals(Type.String))
return distance(StringUtils.left(a.stringValue(), limit), StringUtils.left(b.stringValue(), limit));
return distance(StringUtils.left(a.stringValue(), limit), StringUtils.left(b.stringValue(), limit), conf);
throw new IllegalArgumentException("invalid types\n- A: " + a.toString() + "\n- B: " + b.toString());
}

5
dnet-pace-core/src/main/java/eu/dnetlib/pace/distance/algo/UrlMatcher.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.pace.distance.algo;
import eu.dnetlib.pace.config.Config;
import eu.dnetlib.pace.distance.DistanceClass;
import eu.dnetlib.pace.model.Field;
import org.apache.commons.lang.StringUtils;
@ -28,7 +29,7 @@ public class UrlMatcher extends Levenstein {
}
@Override
public double distance(Field a, Field b) {
public double distance(Field a, Field b, final Config conf) {
final URL urlA = asUrl(getFirstValue(a));
final URL urlB = asUrl(getFirstValue(b));
@ -44,7 +45,7 @@ public class UrlMatcher extends Levenstein {
return hostW * 0.5;
}
return hostW + pathW * super.distance(urlA.getPath(), urlB.getPath());
return hostW + pathW * super.distance(urlA.getPath(), urlB.getPath(), conf);
}
private URL asUrl(final String value) {

39
dnet-pace-core/src/test/java/eu/dnetlib/pace/clustering/ClusteringFunctionTest.java
View File

@ -1,22 +1,25 @@
package eu.dnetlib.pace.clustering;
import java.util.Map;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import eu.dnetlib.pace.AbstractPaceTest;
import eu.dnetlib.pace.common.AbstractPaceFunctions;
import eu.dnetlib.pace.model.Field;
import eu.dnetlib.pace.config.DedupConfig;
import eu.dnetlib.pace.distance.DistanceAlgoTest;
import org.junit.Before;
import org.junit.Test;
import java.util.Map;
public class ClusteringFunctionTest extends AbstractPaceTest {
private Map<String, Integer> params;
DedupConfig conf;
@Before
public void setUp() throws Exception {
params = Maps.newHashMap();
conf = DedupConfig.load(AbstractPaceFunctions.readFromClasspath("/eu/dnetlib/pace/config/org.curr.conf", ClusteringFunctionTest.class));
}
@Test
@ -26,7 +29,7 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
final String s = "http://www.test.it/path/to/resource";
System.out.println(s);
System.out.println(urlClustering.apply(Lists.newArrayList(url(s))));
System.out.println(urlClustering.apply(conf, Lists.newArrayList(url(s))));
}
@Test
@ -40,7 +43,7 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
final String s = "Search for the Standard Model Higgs Boson";
System.out.println(s);
System.out.println(ngram.apply(Lists.newArrayList(title(s))));
System.out.println(ngram.apply(conf, Lists.newArrayList(title(s))));
}
@Test
@ -52,7 +55,7 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
final String s = "Search for the Standard Model Higgs Boson";
System.out.println(s);
System.out.println(np.apply(Lists.newArrayList(title(s))));
System.out.println(np.apply(conf, Lists.newArrayList(title(s))));
}
@Test
@ -64,11 +67,11 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
final String s1 = "University of Pisa";
System.out.println(s1);
System.out.println(np.apply(Lists.newArrayList(title(s1))));
System.out.println(np.apply(conf, Lists.newArrayList(title(s1))));
final String s2 = "Pisa University";
System.out.println(s2);
System.out.println(np.apply(Lists.newArrayList(title(s2))));
System.out.println(np.apply(conf, Lists.newArrayList(title(s2))));
}
@Test
@ -81,7 +84,7 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
final String s = "Search for the Standard Model Higgs Boson";
System.out.println(s);
System.out.println(acro.apply(Lists.newArrayList(title(s))));
System.out.println(acro.apply(conf, Lists.newArrayList(title(s))));
}
@Test
@ -93,7 +96,7 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
final String s = "Search for the Standard Model Higgs Boson";
System.out.println(s);
System.out.println(sp.apply(Lists.newArrayList(title(s))));
System.out.println(sp.apply(conf, Lists.newArrayList(title(s))));
}
@Test
@ -105,7 +108,7 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
final String s = "Search for the Standard Model Higgs Boson";
System.out.println(s);
System.out.println(sp.apply(Lists.newArrayList(title(s))));
System.out.println(sp.apply(conf, Lists.newArrayList(title(s))));
}
@Test
@ -114,7 +117,7 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
final String s = readFromClasspath("gt.author.json");
System.out.println(s);
System.out.println(cf.apply(Lists.newArrayList(person(s))));
System.out.println(cf.apply(conf, Lists.newArrayList(person(s))));
}
@Test
@ -123,27 +126,27 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
final ClusteringFunction cf = new KeywordsClustering(params);
final String s = "Polytechnic University of Turin";
System.out.println(s);
System.out.println(cf.apply(Lists.newArrayList(title(s))));
System.out.println(cf.apply(conf, Lists.newArrayList(title(s))));
final String s1 = "POLITECNICO DI TORINO";
System.out.println(s1);
System.out.println(cf.apply(Lists.newArrayList(title(s1))));
System.out.println(cf.apply(conf, Lists.newArrayList(title(s1))));
final String s2 = "Universita farmaceutica culturale di milano bergamo";
System.out.println("s2 = " + s2);
System.out.println(cf.apply(Lists.newArrayList(title(s2))));
System.out.println(cf.apply(conf, Lists.newArrayList(title(s2))));
final String s3 = "universita universita milano milano";
System.out.println("s3 = " + s3);
System.out.println(cf.apply(Lists.newArrayList(title(s3))));
System.out.println(cf.apply(conf, Lists.newArrayList(title(s3))));
final String s4 = "Politechniki Warszawskiej (Warsaw University of Technology)";
System.out.println("s4 = " + s4);
System.out.println(cf.apply(Lists.newArrayList(title(s4))));
System.out.println(cf.apply(conf, Lists.newArrayList(title(s4))));
final String s5 = "İstanbul Ticarət Universiteti";
System.out.println("s5 = " + s5);
System.out.println(cf.apply(Lists.newArrayList(title(s5))));
System.out.println(cf.apply(conf, Lists.newArrayList(title(s5))));
}

25
dnet-pace-core/src/test/java/eu/dnetlib/pace/distance/DistanceAlgoTest.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.pace.distance;
import eu.dnetlib.pace.clustering.NGramUtils;
import eu.dnetlib.pace.config.DedupConfig;
import eu.dnetlib.pace.distance.algo.JaroWinklerNormalizedName;
import org.junit.Before;
import org.junit.Test;
@ -17,13 +18,13 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
private final static String TEST_STRING = "Toshiba NB550D: è un netbook su piattaforma AMD Fusion⁽¹²⁾.";
private Map<String, Number> params;
private DedupConfig conf;
@Before
public void setup() {
System.out.println("****************************************************************");
System.out.println("Test String : " + TEST_STRING);
params = new HashMap<>();
params.put("weight", 1.0);
conf = DedupConfig.load(readFromClasspath("/eu/dnetlib/pace/config/org.curr.conf", DistanceAlgoTest.class));
}
@Test
@ -55,7 +56,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
@Test
public void testJaroWinklerNormalizedName() {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("Free University of Bozen-Bolzano", "University of the Free State");
double result = jaroWinklerNormalizedName.distance("Free University of Bozen-Bolzano", "University of the Free State", conf);
System.out.println("result = " + result);
assertEquals(0.0, result);
@ -65,7 +66,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
public void testJaroWinklerNormalizedName2() {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("University of New York", "Università di New York");
double result = jaroWinklerNormalizedName.distance("University of New York", "Università di New York", conf);
assertEquals(1.0, result);
}
@ -74,7 +75,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
public void testJaroWinklerNormalizedName3() {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("Biblioteca dell'Universita di Bologna", "Università di Bologna");
double result = jaroWinklerNormalizedName.distance("Biblioteca dell'Universita di Bologna", "Università di Bologna", conf);
System.out.println("result = " + result);
assertEquals(0.0, result);
@ -84,7 +85,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
public void testJaroWinklerNormalizedName4() {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("Universita degli studi di Pisa", "Universita di Pisa");
double result = jaroWinklerNormalizedName.distance("Universita degli studi di Pisa", "Universita di Pisa", conf);
System.out.println("result = " + result);
assertEquals(1.0, result);
@ -94,7 +95,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
public void testJaroWinklerNormalizedName5() {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("RESEARCH PROMOTION FOUNDATION", "IDRYMA PROOTHISIS EREVNAS");
double result = jaroWinklerNormalizedName.distance("RESEARCH PROMOTION FOUNDATION", "IDRYMA PROOTHISIS EREVNAS", conf);
System.out.println("result = " + result);
assertEquals(1.0, result);
@ -104,7 +105,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
public void testJaroWinklerNormalizedName6() {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("Fonds zur Förderung der wissenschaftlichen Forschung (Austrian Science Fund)", "Fonds zur Förderung der wissenschaftlichen Forschung");
double result = jaroWinklerNormalizedName.distance("Fonds zur Förderung der wissenschaftlichen Forschung (Austrian Science Fund)", "Fonds zur Förderung der wissenschaftlichen Forschung", conf);
System.out.println("result = " + result);
assertTrue(result > 0.9);
@ -115,7 +116,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
public void testJaroWinklerNormalizedName7() {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("Polytechnic University of Turin", "POLITECNICO DI TORINO");
double result = jaroWinklerNormalizedName.distance("Polytechnic University of Turin", "POLITECNICO DI TORINO", conf);
System.out.println("result = " + result);
assertTrue(result > 0.9);
@ -125,7 +126,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
public void testJaroWinklerNormalizedName8() {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("Politechniki Warszawskiej (Warsaw University of Technology)", "Warsaw University of Technology");
double result = jaroWinklerNormalizedName.distance("Politechniki Warszawskiej (Warsaw University of Technology)", "Warsaw University of Technology", conf);
System.out.println("result = " + result);
}
@ -134,7 +135,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
public void testJaroWinklerNormalizedName9() {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("Istanbul Commerce University", "İstanbul Ticarət Universiteti");
double result = jaroWinklerNormalizedName.distance("Istanbul Commerce University", "İstanbul Ticarət Universiteti", conf);
System.out.println("result = " + result);
}
@ -144,7 +145,7 @@ public class DistanceAlgoTest extends AbstractPaceFunctions {
final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
double result = jaroWinklerNormalizedName.distance("Firenze University Press", "University of Florence");
double result = jaroWinklerNormalizedName.distance("Firenze University Press", "University of Florence", conf);
System.out.println("result = " + result);
}

124
dnet-pace-core/src/test/resources/eu/dnetlib/pace/config/org.curr.conf
View File

@ -5,7 +5,7 @@
"entityType" : "organization",
"orderField" : "legalname",
"queueMaxSize" : "2000",
"groupMaxSize" : "10",
"groupMaxSize" : "50",
"slidingWindowSize" : "200",
"rootBuilder" : [ "organization", "projectOrganization_participation_isParticipant", "datasourceOrganization_provision_isProvidedBy" ],
"includeChildren" : "true"
@ -14,23 +14,131 @@
"clustering" : [
{ "name" : "sortedngrampairs", "fields" : [ "legalname" ], "params" : { "max" : 2, "ngramLen" : "3"} },
{ "name" : "suffixprefix", "fields" : [ "legalname" ], "params" : { "max" : 1, "len" : "3" } },
{ "name" : "urlclustering", "fields" : [ "websiteurl" ], "params" : { } }
{ "name" : "urlclustering", "fields" : [ "websiteurl" ], "params" : { } },
{ "name" : "keywordsclustering", "fields" : [ "legalname" ], "params" : { "max": 2, "windowSize": 4} }
],
"strictConditions" : [
{ "name" : "exactMatch", "fields" : [ "gridid" ] }
],
"conditions" : [
{ "name" : "exactMatch", "fields" : [ "country" ] },
{ "name" : "DomainExactMatch", "fields" : [ "websiteurl" ] }
{ "name" : "DomainExactMatch", "fields" : [ "websiteurl" ] },
{ "name" : "exactMatch", "fields" : [ "country" ] }
],
"model" : [
{ "name" : "legalname", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value" },
{ "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/country/classid" },
{ "name" : "country", "algo" : "Null", "type" : "String", "weight" : "0", "ignoreMissing" : "false", "path" : "organization/metadata/country/classid" },
{ "name" : "legalshortname", "algo" : "JaroWinklerNormalizedName", "type" : "String", "weight" : "0.1", "ignoreMissing" : "true", "path" : "organization/metadata/legalshortname/value" },
{ "name" : "legalname", "algo" : "JaroWinklerNormalizedName", "type" : "String", "weight" : "0.9", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value", "params" : {"windowSize" : 4, "threshold" : 0.5} },
{ "name" : "legalname", "algo" : "JaroWinklerNormalizedName", "type" : "String", "weight" : "0.9", "ignoreMissing" : "false", "path" : "organization/metadata/legalname/value", "params" : {"windowSize" : 4, "threshold" : 0.7} },
{ "name" : "websiteurl", "algo" : "Null", "type" : "URL", "weight" : "0", "ignoreMissing" : "true", "path" : "organization/metadata/websiteurl/value", "params" : { "host" : 0.5, "path" : 0.5 } },
{ "name" : "gridid", "algo" : "Null", "type" : "String", "weight" : "0.0", "ignoreMissing" : "true", "path" : "pid[qualifier#classid = {grid}]/value" }
],
"blacklists" : { }
"blacklists" : {
"legalname" : []
},
"synonyms": {
"key::1": ["university","università","universita","università studi","universita studi","universitario","universitaria","université","universitaire","universitaires","universidad","universitade","Universität","universitaet","Uniwersytet","университет","universiteit","πανεπιστήμιο","universitesi","universiteti"],
"key::2": ["studies","studi","études","estudios","estudos","Studien","studia","исследования","studies","σπουδές"],
"key::3": ["advanced","superiore","supérieur","supérieure","supérieurs","supérieures","avancado","avancados","fortgeschrittene","fortgeschritten","zaawansowany","передовой","gevorderd","gevorderde","προχωρημένος","προχωρημένη","προχωρημένο","προχωρημένες","προχωρημένα","wyzsza"],
"key::4": ["institute","istituto","institut","instituto","instituto","Institut","instytut","институт","instituut","ινστιτούτο"],
"key::5": ["hospital","ospedale","hôpital","hospital","hospital","Krankenhaus","szpital","больница","ziekenhuis","νοσοκομείο"],
"key::6": ["research","ricerca","recherche","investigacion","pesquisa","Forschung","badania","исследования","onderzoek","έρευνα","erevna","erevnas"],
"key::7": ["college","collegio","université","colegio","faculdade","Hochschule","Szkoła Wyższa","Высшая школа","universiteit","κολλέγιο"],
"key::8": ["foundation","fondazione","fondation","fundación","fundação","Stiftung","Fundacja","фонд","stichting","ίδρυμα","idryma"],
"key::9": ["center","centro","centre","centro","centro","zentrum","centrum","центр","centrum","κέντρο"],
"key::10": ["national","nazionale","national","nationale","nationaux","nationales","nacional","nacional","national","krajowy","национальный","nationaal","nationale","εθνικό"],
"key::11": ["association","associazione","association","asociación","associação","Verein","verband","stowarzyszenie","ассоциация","associatie"],
"key::12": ["society","societa","société","sociedad","sociedade","gesellschaft","społeczeństwo","общество","maatschappij","κοινωνία"],
"key::13": ["international","internazionale","international","internacional","internacional","international","międzynarodowy","Международный","internationaal","internationale","διεθνής","διεθνή","διεθνές"],
"key::14": ["community","comunita","communauté","comunidad","comunidade","Gemeinschaft","społeczność","сообщество","gemeenschap","κοινότητα"],
"key::15": ["school","scuola","école","escuela","escola","schule","Szkoła","школа","school","σχολείο"],
"key::16": ["education","educazione","éducation","educacion","Educação","Bildung","Edukacja","образование","opleiding","εκπαίδευση"],
"key::17": ["academy","accademia","académie","academia","academia","Akademie","akademie","академия","academie","ακαδημία"],
"key::18": ["public","pubblico","public","publique","publics","publiques","publico","publico","Öffentlichkeit","publiczny","публичный","publiek","publieke","δημόσιος","δημόσια","δημόσιο"],
"key::19": ["museum","museo","musée","mueso","museu","museum","muzeum","музей","museum","μουσείο"],
"key::20": ["group","gruppo","groupe","grupo","grupo","gruppe","grupa","группа","groep","ομάδα","όμιλος"],
"key::21": ["department","dipartimento","département","departamento","departamento","abteilung","departament","отдел","afdeling","τμήμα"],
"key::22": ["council","consiglio","conseil","Consejo","conselho","gesellschaft","rada","совет","raad","συμβούλιο"],
"key::23": ["library","biblioteca","bibliothèque","biblioteca","biblioteca","Bibliothek","biblioteka","библиотека","bibliotheek","βιβλιοθήκη"],
"key::24": ["ministry","ministero","ministère","ministerio","ministério","Ministerium","ministerstwo","министерство","ministerie","υπουργείο"],
"key::25": ["services","servizi","services","servicios","Serviços","Dienstleistungen","usługi","услуги","diensten","υπηρεσίες"],
"key::26": ["central","centrale","central","centrale","centrales","central","central","zentral","centralny","цетральный","centraal","κεντρικός","κεντρική","κεντρικό","κεντρικά"],
"key::27": ["general","generale","général","générale","généraux","générales","general","geral","general","Allgemeines","general","общий","algemeen","algemene","γενικός","γενική","γενικό","γενικά"],
"key::28": ["applied","applicati","appliqué","appliquée","appliqués","appliquées","aplicado","aplicada","angewendet","stosowany","прикладной","toegepast","toegepaste","εφαρμοσμένος","εφαρμοσμένη","εφαρμοσμένο","εφαρμοσμένα"],
"key::29": ["european","europee","europea","européen","européenne","européens","européennes","europeo","europeu","europäisch","europejski","европейский","Europees","Europese","ευρωπαϊκός","ευρωπαϊκή","ευρωπαϊκό","ευρωπαϊκά"],
"key::30": ["agency","agenzia","agence","agencia","agencia","agentur","agencja","агенция","agentschap","πρακτορείο"],
"key::31": ["laboratory","laboratorio","laboratoire","laboratorio","laboratorio","labor","laboratorium","лаборатория","laboratorium","εργαστήριο"],
"key::32": ["industry","industria","industrie","индустрия","industrie","βιομηχανία"],
"key::33": ["industrial","industriale","industriel","industrielle","industriels","industrielles","индустриальный","industrieel","βιομηχανικός","βιομηχανική","βιομηχανικό","βιομηχανικά","βιομηχανικές"],
"key::34": ["consortium","consorzio","consortium","консорциум","consortium","κοινοπραξία"],
"key::35": ["organization","organizzazione","organisation","organización","organização","organizacja","организация","organisatie","οργανισμός"],
"key::36": ["authority","autorità","autorité","авторитет","autoriteit"],
"key::37": ["federation","federazione","fédération","федерация","federatie","ομοσπονδία"],
"key::38": ["observatory","osservatorio","observatoire","обсерватория","observatorium","αστεροσκοπείο"],
"key::39": ["bureau","ufficio","bureau","офис","bureau","γραφείο"],
"key::40": ["company","impresa","compagnie","société","компания","bedrijf","εταιρία"],
"key::41": ["polytechnic","politecnico","polytechnique","политехника","polytechnisch","πολυτεχνείο","universita politecnica","polytechnic university","universidad politecnica","universitat politecnica","politechnika","politechniki","university technology","university science technology"],
"key::42": ["coalition","coalizione","coalition","коалиция","coalitie","συνασπισμός"],
"key::43": ["initiative","iniziativa","initiative","инициатива","initiatief","πρωτοβουλία"],
"key::44": ["academic","accademico","académique","universitaire","акадеческий academisch","ακαδημαϊκός","ακαδημαϊκή","ακαδημαϊκό","ακαδημαϊκές","ακαδημαϊκοί"],
"key::45": ["institution","istituzione","institution","институциональный","instelling","ινστιτούτο"],
"key::46": ["division","divisione","division","отделение","divisie","τμήμα"],
"key::47": ["committee","comitato","comité","комитет","commissie","επιτροπή"],
"key::48": ["promotion","promozione","продвижение","proothisis","forderung"],
"key::49": ["medical","medicine","clinical","medicina","clinici","médico","medicina","clínica","médico","medicina","clínica","medizinisch","Medizin","klinisch","medisch","geneeskunde","klinisch","ιατρικός","ιατρική","ιατρικό","ιατρικά","κλινικός","κλινική","κλινικό","κλινικά","tıbbi","tıp","klinik","orvosi","orvostudomány","klinikai","zdravniški","medicinski","klinični","meditsiini","kliinik","kliiniline"],
"key::50": ["technology","technological","tecnologia","tecnologie","tecnología","tecnológico","tecnologia","tecnológico","Technologie","technologisch","technologie","technologisch","τεχνολογία","τεχνολογικός","τεχνολογική","τεχνολογικό","teknoloji","teknolojik","technológia","technológiai","tehnologija","tehnološki","tehnoloogia","tehnoloogiline","technologii","technical","texniki","teknik"],
"key::51": ["science","scientific","scienza","scientifiche","scienze","ciencia","científico","ciência","científico","Wissenschaft","wissenschaftlich","wetenschap","wetenschappelijk","επιστήμη","επιστημονικός","επιστημονική","επιστημονικό","επιστημονικά","bilim","bilimsel","tudomány","tudományos","znanost","znanstveni","teadus","teaduslik"],
"key::52": ["engineering","ingegneria","ingeniería","engenharia","Ingenieurwissenschaft","ingenieurswetenschappen","bouwkunde","μηχανικός","μηχανική","μηχανικό","mühendislik","mérnöki","Inženirstvo","inseneeria","inseneri"],
"key::53": ["management","gestione","gestionale","gestionali","gestión","administración","gestão","administração","Verwaltung","management","διαχείριση","yönetim","menedzsment","vodstvo","upravljanje","management","juhtkond","juhtimine","haldus"],
"key::54": ["energy","energia","energía","energia","Energie","energie","ενέργεια","enerji","energia","energija","energia"],
"key::55": ["agricultural","agriculture","agricoltura","agricole","agrícola","agricultura","agrícola","agricultura","landwirtschaftlich","Landwirtschaft","landbouwkundig","landbouw","αγροτικός","αγροτική","αγροτικό","γεωργικός","γεωργική","γεωργικό","γεωργία","tarımsal","tarım","mezőgazdasági","mezőgazdaság","poljedelski","poljedelstvo","põllumajandus","põllumajanduslik"],
"key::56": ["information","informazione","información","informação","Information","informatie","πληροφορία","bilgi","információ","informacija","informatsioon","informatycznych"],
"key::57": ["social","sociali","social","social","Sozial","sociaal","maatschappelijk","κοινωνικός","κοινωνική","κοινωνικό","κοινωνικά","sosyal","szociális","družbeni","sotsiaal","sotsiaalne"],
"key::58": ["environmental","ambiente","medioambiental","ambiente","medioambiente","meioambiente","Umwelt","milieu","milieuwetenschap","milieukunde","περιβαλλοντικός","περιβαλλοντική","περιβαλλοντικό","περιβαλλοντικά","çevre","környezeti","okoliški","keskonna"],
"key::59": ["business","economia","economiche","economica","negocio","empresa","negócio","Unternehmen","bedrijf","bedrijfskunde","επιχείρηση","iş","üzleti","posel","ettevõte/äri"],
"key::60": ["pharmaceuticals","pharmacy","farmacia","farmaceutica","farmacéutica","farmacia","farmacêutica","farmácia","Pharmazeutika","Arzneimittelkunde","farmaceutica","geneesmiddelen","apotheek","φαρμακευτικός","φαρμακευτική","φαρμακευτικό","φαρμακευτικά","φαρμακείο","ilaç","eczane","gyógyszerészeti","gyógyszertár","farmacevtika","lekarništvo","farmaatsia","farmatseutiline"],
"key::61": ["healthcare","health services","salute","atenciónmédica","cuidadodelasalud","cuidadoscomasaúde","Gesundheitswesen","gezondheidszorg","ιατροφαρμακευτικήπερίθαλψη","sağlıkhizmeti","egészségügy","zdravstvo","tervishoid","tervishoiu"],
"key::62": ["history","storia","historia","história","Geschichte","geschiedenis","geschiedkunde","ιστορία","tarih","történelem","zgodovina","ajalugu"],
"key::63": ["materials","materiali","materia","materiales","materiais","materialen","υλικά","τεκμήρια","malzemeler","anyagok","materiali","materjalid","vahendid"],
"key::64": ["economics","economia","economiche","economica","economía","economia","Wirtschaft","economie","οικονομικά","οικονομικέςεπιστήμες","ekonomi","közgazdaságtan","gospodarstvo","ekonomija","majanduslik","majandus"],
"key::65": ["therapeutics","terapeutica","terapéutica","terapêutica","therapie","θεραπευτική","tedavibilimi","gyógykezelés","terapevtika","terapeutiline","ravi"],
"key::66": ["oncology","oncologia","oncologico","oncología","oncologia","Onkologie","oncologie","ογκολογία","onkoloji","onkológia","onkologija","onkoloogia"],
"key::67": ["natural","naturali","naturale","natural","natural","natürlich","natuurlijk","φυσικός","φυσική","φυσικό","φυσικά","doğal","természetes","naraven","loodus"],
"key::68": ["educational","educazione","pedagogia","educacional","educativo","educacional","pädagogisch","educatief","εκπαιδευτικός","εκπαιδευτική","εκπαιδευτικό","εκπαιδευτικά","eğitimsel","oktatási","izobraževalen","haridus","hariduslik"],
"key::69": ["biomedical","biomedica","biomédico","biomédico","biomedizinisch","biomedisch","βιοιατρικός","βιοιατρική","βιοιατρικό","βιοιατρικά","biyomedikal","orvosbiológiai","biomedicinski","biomeditsiiniline"],
"key::70": ["veterinary","veterinaria","veterinarie","veterinaria","veterinária","tierärtzlich","veterinair","veeartsenijlkunde","κτηνιατρικός","κτηνιατρική","κτηνιατρικό","κτηνιατρικά","veteriner","állatorvosi","veterinar","veterinarski","veterinaaria"],
"key::71": ["chemistry","chimica","química","química","Chemie","chemie","scheikunde","χημεία","kimya","kémia","kemija","keemia"],
"key::72": ["security","sicurezza","seguridad","segurança","Sicherheit","veiligheid","ασφάλεια","güvenlik","biztonsági","varnost","turvalisus","julgeolek"],
"key::73": ["biotechnology","biotecnologia","biotecnologie","biotecnología","biotecnologia","Biotechnologie","biotechnologie","βιοτεχνολογία","biyoteknoloji","biotechnológia","biotehnologija","biotehnoloogia"],
"key::74": ["military","militare","militari","militar","militar","Militär","militair","leger","στρατιωτικός","στρατιωτική","στρατιωτικό","στρατιωτικά","askeri","katonai","vojaški","vojni","militaar","wojskowa"],
"key::75": ["theological","teologia","teologico","teológico","tecnológica","theologisch","theologisch","θεολογικός","θεολογική","θεολογικό","θεολογικά","teolojik","technológiai","teološki","teoloogia","usuteadus","teoloogiline"],
"key::76": ["electronics","elettronica","electrónica","eletrônicos","Elektronik","elektronica","ηλεκτρονική","elektronik","elektronika","elektronika","elektroonika"],
"key::77": ["forestry","forestale","forestali","silvicultura","forestal","floresta","Forstwirtschaft","bosbouw","δασοκομία","δασολογία","ormancılık","erdészet","gozdarstvo","metsandus"],
"key::78": ["maritime","marittima","marittime","marittimo","marítimo","marítimo","maritiem","ναυτικός","ναυτική","ναυτικό","ναυτικά","ναυτιλιακός","ναυτιλιακή","ναυτιλιακό","ναυτιλιακά","θαλάσσιος","θαλάσσια","θαλάσσιο","denizcilik","tengeri","morski","mere","merendus"],
"key::79": ["sports","sport","deportes","esportes","Sport","sport","sportwetenschappen","άθληση","γυμναστικήδραστηριότητα","spor","sport","šport","sport","spordi"],
"key::80": ["surgery","chirurgia","chirurgiche","cirugía","cirurgia","Chirurgie","chirurgie","heelkunde","εγχείρηση","επέμβαση","χειρουργικήεπέμβαση","cerrahi","sebészet","kirurgija","kirurgia"],
"key::81": ["cultural","culturale","culturali","cultura","cultural","cultural","kulturell","cultureel","πολιτιστικός","πολιτιστική","πολιτιστικό","πολιτισμικός","πολιτισμική","πολιτισμικό","kültürel","kultúrális","kulturni","kultuuri","kultuuriline"],
"key::82": ["computerscience","informatica","ordenador","computadora","informática","computación","cienciasdelacomputación","ciênciadacomputação","Computer","computer","υπολογιστής","ηλεκτρονικόςυπολογιστής","bilgisayar","számítógép","računalnik","arvuti"],
"key::83": ["finance","financial","finanza","finanziarie","finanza","financiero","finanças","financeiro","Finanzen","finanziell","financiën","financieel","χρηματοοικονομικά","χρηματοδότηση","finanse","finansal","pénzügy","pénzügyi","finance","finančni","finants","finantsiline"],
"key::84": ["communication","comunicazione","comuniciación","comunicação","Kommunikation","communication","επικοινωνία","iletişim","kommunikáció","komuniciranje","kommunikatsioon"],
"key::85": ["justice","giustizia","justicia","justiça","Recht","Justiz","justitie","gerechtigheid","δικαιοσύνη","υπουργείοδικαιοσύνης","δίκαιο","adalet","igazságügy","pravo","õigus"],
"key::86": ["aerospace","aerospaziale","aerospaziali","aeroespacio","aeroespaço","Luftfahrt","luchtvaart","ruimtevaart","αεροπορικός","αεροπορική","αεροπορικό","αεροναυπηγικός","αεροναυπηγική","αεροναυπηγικό","αεροναυπηγικά","havacılıkveuzay","légtér","zrakoplovstvo","atmosfäär","kosmos"],
"key::87": ["dermatology","dermatologia","dermatología","dermatologia","Dermatologie","dermatologie","δρματολογία","dermatoloji","bőrgyógyászat","dermatológia","dermatologija","dermatoloogia"],
"key::88": ["architecture","architettura","arquitectura","arquitetura","Architektur","architectuur","αρχιτεκτονική","mimarlık","építészet","arhitektura","arhitektuur"],
"key::89": ["mathematics","matematica","matematiche","matemáticas","matemáticas","Mathematik","wiskunde","mathematica","μαθηματικά","matematik","matematika","matematika","matemaatika"],
"key::90": ["language","lingue","linguistica","linguistiche","lenguaje","idioma","língua","idioma","Sprache","taal","taalkunde","γλώσσα","dil","nyelv","jezik","keel"],
"key::91": ["neuroscience","neuroscienza","neurociencia","neurociência","Neurowissenschaft","neurowetenschappen","νευροεπιστήμη","nörobilim","idegtudomány","nevroznanost","neuroteadused"],
"key::92": ["automation","automazione","automatización","automação","Automatisierung","automatisering","αυτοματοποίηση","otomasyon","automatizálás","avtomatizacija","automatiseeritud"],
"key::93": ["pediatric","pediatria","pediatriche","pediatrico","pediátrico","pediatría","pediátrico","pediatria","pädiatrisch","pediatrische","παιδιατρική","pediatrik","gyermekgyógyászat","pediatrija","pediaatria"],
"key::94": ["photonics","fotonica","fotoniche","fotónica","fotônica","Photonik","fotonica","φωτονική","fotonik","fotonika","fotonika","fotoonika"],
"key::95": ["mechanics","meccanica","meccaniche","mecánica","mecânica","Mechanik","Maschinenbau","mechanica","werktuigkunde","μηχανικής","mekanik","gépészet","mehanika","mehaanika"],
"key::96": ["psychiatrics","psichiatria","psichiatrica","psichiatriche","psiquiatría","psiquiatria","Psychiatrie","psychiatrie","ψυχιατρική","psikiyatrik","pszihiátria","psihiatrija","psühhaatria"],
"key::97": ["psychology","fisiologia","psicología","psicologia","Psychologie","psychologie","ψυχολογία","psikoloji","pszihológia","psihologija","psühholoogia"],
"key::98": ["automotive","industriaautomobilistica","industriadelautomóvil","automotriz","industriaautomotriz","automotivo","Automobilindustrie","autoindustrie","αυτοκίνητος","αυτοκίνητη","αυτοκίνητο","αυτοκινούμενος","αυτοκινούμενη","αυτοκινούμενο","αυτοκινητιστικός","αυτοκινητιστική","αυτοκινητιστικό","otomotiv","autóipari","samogiben","avtomobilskaindustrija","auto-"],
"key::99": ["neurology","neurologia","neurologiche","neurología","neurologia","Neurologie","neurologie","zenuwleer","νευρολογία","nöroloji","neurológia","ideggyógyászat","nevrologija","neuroloogia"],
"key::100": ["geology","geologia","geologiche","geología","geologia","Geologie","geologie","aardkunde","γεωλογία","jeoloji","geológia","földtudomány","geologija","geoloogia"],
"key::101": ["microbiology","microbiologia","micro-biologia","microbiologiche","microbiología","microbiologia","Mikrobiologie","microbiologie","μικροβιολογία","mikrobiyoloji","mikrobiológia","mikrobiologija","mikrobioloogia"],
"key::102": ["informatics","informatica","informática","informática","informatica"],
"key::103": ["forschungsgemeinschaft","comunita ricerca","research community","research foundation","research association"],
"key::104": ["commerce","ticaret","ticarət","commercio","trade","handel","comercio"]
}
}
}