configuration updated for testing

dnet-dedup-test/src/test/resources/eu/dnetlib/pace/config/authors.fdup.conf.json

@@ -5,7 +5,7 @@
     "entityType": "author",
     "subEntityType": "author",
     "subEntityValue": "author",
-    "orderField": "fullname",
+    "orderField": "name",
     "queueMaxSize": "200",
     "groupMaxSize": "100",
     "maxChildren": "100",

dnet-dedup-test/src/test/resources/eu/dnetlib/pace/config/authors.fdup.soft.conf.json

@@ -5,7 +5,7 @@
     "entityType": "author",
     "subEntityType": "author",
     "subEntityValue": "author",
-    "orderField": "fullname",
+    "orderField": "name",
     "queueMaxSize": "200",
     "groupMaxSize": "100",
     "maxChildren": "100",

dnet-dedup-test/src/test/resources/eu/dnetlib/pace/config/publication.current.conf.json

@@ -6,9 +6,9 @@
     "subEntityType": "resulttype",
     "subEntityValue": "publication",
     "orderField": "title",
-    "queueMaxSize": "5000",
-    "groupMaxSize": "2000",
-    "maxChildren": "1000",
+    "queueMaxSize": "200",
+    "groupMaxSize": "100",
+    "maxChildren": "100",
     "slidingWindowSize": "50",
     "rootBuilder": [
       "result",
@@ -28,9 +28,26 @@
     "idPath": "$.id"
   },
   "pace": {
-    "clustering" : [
-      { "name" : "wordsStatsSuffixPrefixChain", "fields" : [ "title" ], "params" : { "mod" : "10" } },
-      { "name" : "lowercase", "fields" : [ "doi" ], "params" : { } }
+    "clustering": [
+      {
+        "name": "wordsStatsSuffixPrefixChain",
+        "fields": [
+          "title"
+        ],
+        "params": {
+          "mod": "10"
+        }
+      },
+      {
+        "name": "lowercase",
+        "fields": [
+          "doi",
+          "altdoi"
+        ],
+        "params": {
+          "collapseOn:pid": "0"
+        }
+      }
     ],
     "decisionTree": {
       "start": {
@@ -42,18 +59,75 @@
             "countIfUndefined": "false",
             "params": {
               "jpath_value": "$.value",
-              "jpath_classid": "$.qualifier.classid"
+              "jpath_classid": "$.qualifier.classid",
+              "mode": "count"
             }
           }
         ],
-        "threshold": 0.5,
-        "aggregation": "AVG",
+        "threshold": 1.0,
+        "aggregation": "MAX",
         "positive": "MATCH",
-        "negative": "layer2",
-        "undefined": "layer2",
+        "negative": "instanceTypeCheck",
+        "undefined": "instanceTypeCheck",
+        "ignoreUndefined": "false"
+      },
+      "instanceTypeCheck": {
+        "fields": [
+          {
+            "field": "instance",
+            "comparator": "instanceTypeMatch",
+            "weight": 1.0,
+            "countIfUndefined": "false",
+            "params": {}
+          }
+        ],
+        "threshold": 0.5,
+        "aggregation": "MAX",
+        "positive": "pidVSaltid",
+        "negative": "NO_MATCH",
+        "undefined": "pidVSaltid",
         "ignoreUndefined": "true"
       },
-      "layer2": {
+      "pidVSaltid": {
+        "fields": [
+          {
+            "field": "pid",
+            "comparator": "jsonListMatch",
+            "weight": 1.0,
+            "countIfUndefined": "false",
+            "params": {
+              "jpath_value": "$.value",
+              "jpath_classid": "$.qualifier.classid",
+              "crossCompare": "alternateid",
+              "mode": "count"
+            }
+          }
+        ],
+        "threshold": 1.0,
+        "aggregation": "MAX",
+        "positive": "softCheck",
+        "negative": "earlyExits",
+        "undefined": "earlyExits",
+        "ignoreUndefined": "true"
+      },
+      "softCheck": {
+        "fields": [
+          {
+            "field": "title",
+            "comparator": "levensteinTitle",
+            "weight": 1.0,
+            "countIfUndefined": "true",
+            "params": {}
+          }
+        ],
+        "threshold": 0.9,
+        "aggregation": "AVG",
+        "positive": "MATCH",
+        "negative": "NO_MATCH",
+        "undefined": "NO_MATCH",
+        "ignoreUndefined": "true"
+      },
+      "earlyExits": {
         "fields": [
           {
             "field": "title",
@@ -72,12 +146,12 @@
         ],
         "threshold": 1.0,
         "aggregation": "AND",
-        "positive": "layer3",
+        "positive": "strongCheck",
         "negative": "NO_MATCH",
-        "undefined": "layer3",
+        "undefined": "strongCheck",
         "ignoreUndefined": "false"
       },
-      "layer3": {
+      "strongCheck": {
         "fields": [
           {
             "field": "title",
@@ -89,28 +163,60 @@
         ],
         "threshold": 0.99,
         "aggregation": "AVG",
-        "positive": "MATCH",
+        "positive": "surnames",
         "negative": "NO_MATCH",
         "undefined": "NO_MATCH",
         "ignoreUndefined": "true"
+      },
+      "surnames": {
+        "fields": [
+          {
+            "field": "authors",
+            "comparator": "authorsMatch",
+            "weight": 1.0,
+            "countIfUndefined": "false",
+            "params": {
+              "surname_th": 0.75,
+              "fullname_th": 0.75,
+              "mode": "full"
+            }
+          }
+        ],
+        "threshold": 0.6,
+        "aggregation": "MAX",
+        "positive": "MATCH",
+        "negative": "NO_MATCH",
+        "undefined": "MATCH",
+        "ignoreUndefined": "true"
       }
     },
     "model": [
       {
         "name": "doi",
         "type": "String",
-        "path": "$.pid[?(@.qualifier.classid == 'doi')].value"
+        "path": "$.instance[*].pid[?(@.qualifier.classid == 'doi')].value"
+      },
+      {
+        "name": "altdoi",
+        "type": "String",
+        "path": "$.instance[*].alternateIdentifier[?(@.qualifier.classid == 'doi')].value"
       },
       {
         "name": "pid",
         "type": "JSON",
-        "path": "$.pid",
+        "path": "$.instance[*].pid[*]",
+        "overrideMatch": "true"
+      },
+      {
+        "name": "alternateid",
+        "type": "JSON",
+        "path": "$.instance[*].alternateIdentifier[*]",
         "overrideMatch": "true"
       },
       {
         "name": "title",
-        "type": "String",
-        "path": "$.title[?(@.qualifier.classid == 'main title')].value",
+        "type": "StringConcat",
+        "path": "$.title[?(@.qualifier.classid == 'main title')].value|||$.title[?(@.qualifier.classid == 'subtitle')].value",
         "length": 250,
         "size": 5
       },
@@ -124,6 +230,11 @@
         "name": "resulttype",
         "type": "String",
         "path": "$.resulttype.classid"
+      },
+      {
+        "name": "instance",
+        "type": "List",
+        "path": "$.instance[*].instancetype.classname"
       }
     ],
     "blacklists": {
@@ -354,7 +465,16 @@
         "^(Measurement of the top quark\\-pair production cross section with ATLAS in pp collisions at).*$",
         "^(Measurement of the spin\\-dependent structure function).*",
         "(?i)^.*authors['’′]? reply\\.?$",
-        "(?i)^.*authors['’′]? response\\.?$"
+        "(?i)^.*authors['’′]? response\\.?$",
+        "^Data [mM]anagement [sS]ervices\\.$",
+        "Research and Advanced Technology for Digital Libraries",
+        "(?i)^risky business$",
+        "(?i)^great expectations\\.?$",
+        "(?i)^what's in a name\\?$",
+        "(?i)^decisions, decisions\\.?$",
+        "(?i)^update to our reader, reviewer, and author communities.*",
+        "(?i)^lest we forget$",
+        "(?i)^measure for measure$"
       ]
     },
     "synonyms": {}