From 12756f9d41f6b12ba2afdd6439d2776016cedae7 Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Wed, 13 May 2020 16:11:40 +0200 Subject: [PATCH] multithread (4 threads) test to feed elastic search --- .../doiboost/orcid/ElasticSearchTest.java | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/ElasticSearchTest.java b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/ElasticSearchTest.java index 0908299b0..b4b2c7844 100644 --- a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/ElasticSearchTest.java +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/ElasticSearchTest.java @@ -11,6 +11,18 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.junit.jupiter.api.Test; +import java.net.ConnectException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + public class ElasticSearchTest { private static final String BASE_CFG_URL = "http://%s:9200/%s/%s/%s?pretty"; @@ -21,6 +33,11 @@ public class ElasticSearchTest { private String record; private int readTimeout = 30000; + private int nThreads = 4; + private int nTasks = 150; + private ExecutorService executorService = Executors.newFixedThreadPool(nThreads); + private List> resList = Lists.newArrayList(); + public void setup() { indexHost = "ip-90-147-167-25.ct1.garrservices.it"; indexName = "orcid_update"; @@ -83,4 +100,86 @@ public class ElasticSearchTest { } } } + +// @Test + public void testMultiThreadFeed() throws Exception { + setup(); + int countAll = 0; + int countOk = 0; + int partial = 0; + String recordTemplate = "{\n" + + " \"timestamp\": 1540825815212,\n" + + " \"pid\": \"%s\",\n" + + " \"blob\": \"H4sIAAAAAAAAAO19zXLcOLbm/j4Fw4spdYSRAkiQBDQuV9iyyla3y65rqW7/bCZAEpR4nUnmJTPtUq36HWZ1I2Z2s+l36F29ST/JAGQyCclCiqJIZjKT3SXbygRxDvFzzvcdHAAvfvh1NjW+8DSLkvj7Z2gCnxk89pMgiq++f/bL5Y+APDOyBYsDNk1i/v2zG549++Hlv71IuZ+kwUnxlzFni+vvnx1D8T8gfkxgmwgD7NK/PDOEgDg7ieIFT2M2/f7Z9WIxPzk+/vr16yRJ/SgQf14dx9lxWaJ8ggdLny1ytXSPrIuUzwRRtohif/NTSqH1c3wu3oQteLDhsXWZ8qlkcc1TELMZ1z9VlSmfmvGZJ9r7Oprrn6rKrFsjTZN0Q0vIr8uyfjKbbWqA4vuydNGF+tLF92XpuVArEb0EAr5g0TTTP3e3ZFnDZ37zdaPAVYH1m8/E0xveXH69LvtrMYZAFPB4EYUR39Rm3xYu6wmXcTEDdM+uCqxbJeUhT8XE4ZsapCpUPseCIOXZhmdWBapZ9CUSAxDMkyzaPMjvlixrEC37Wf+U/LYseS2mSZLe6AuvClT9NJ8mNzPRlJs6qyxTPvVfSzYVTf/QTL9VrHw24+mXyN8w+VYFbo/dh0bsum+EhRBtGG3q0qpMNZsyzlJfTvplumHY3i5XaSh+T/mXiH/dpOa6UPmct5xu6FX57V39RAVZskw3td43RZ+9/DdD/O9FYT9O8sLK5Cm+VUss0+ilrD0T1VdV3+MlXhwrT3xTjfQuLzc9lRf45rHrJFu8XItdF84/Ll7keOObvFAmrDrDFUlqiWnisyl/yeMXx/d8vBK4scoXqxlVTj1FUPmNn/J8CoAZX1wnwcs3kTDPixfHuu+/rSFberMok+4eBMKZvTQhcgG0ATIvITnB9olNJo5j/q2q8+4T3zT0lGULMBOYQbRdUNZqQlkrdC8hPBH/mXCCkPO3dZPf88w9bztl0YwHLxfpkivvuPr02/ICxxT15U7hzmN3vtzw9DyNZiy92VzL7UKr7r23A18UdmVlfjZgpdIA1WxhRAESVdBL0zzB5AS6E4pRzRZeqSRxifElyiIvmkYLYevnS28a+c9WSt6H56pKVAXzcaeophlS95W+t777X7isFYkhZZ6YeGLDmkNKeekKkZxcRV94nIOz7OVPLI2Y8SnJ2ItvoMutgg/UGbJZNL3Jy758FS+Sq5j99tt9daoFH6hTtFcQLYqilZ7GxurVZ6p+P1Y6Xvm4QqkKqM02DVWl2LNjpabbfu2OO9xY452imoFWcyY42HzEwNiotDFfLoBgRMJTIoIdE5nPNkyZOq92vK5HUJH5lN2AKA74r2K+3XnpTTNs8yvrZ1jXjanWX8CGb7//tgzwp5HwvsIH319afeJ+VFFUcPzq55+B/Zd3/355Zlo//+J+fHVxSe9HF7rqczCxsZrbcENXz4Pw45sHjx/VJHcaMJ/RfxKc5LvMOE2mU37FjfdJHCTx3Xq/tTj3SL+nwJ1ZIn42CDXmaRJGUy6mxZVwn7pn60hZd/ZnfzpfpnzJFhPxzwnzJ8vPx/MkXbDpMY9Xpk2QZ2khQSosJGBrC3kUEjPwCA/FkBXQBHs+BAyGCPguDngQWA7x0R8m14vZ9D5t7xiMuyVu/6ra2832UCmZ44iC7G40kwXfVU3uiiOWZHKz3S4JpVrBinCXzHzj82WZWxXcw6Tvo+Kb3+ue8k/zAbXR0GPeQfEG1LRsCh/tDO570eOyric4BAIgAojI97cF6iITZJGOHMKjWlatf3QI++QQVt8rAxosbub85YWfzJeZ8WopKGhqnL9Z1/RNyVp1fmHTJX9p26ZJHYRdCO+tryhVq8LSp6yCHlmu7kQUPGZC5Sg4LjD0RJT7geUvcR58/5D8b5yETnrKpzlFl9HdlxmfhvdWd6vUHedT01SpbqiuhV7HKVTKuvqwCnYpsTGQLWeSB2/0OVWUrHHwwLZJTWqrqKlE+jf6HrXcbdda1bVea9jsxdaltPWsQ6APuPl1scd6QVOY69wL0BNkTdxHMSG1j0OB36IimnSVJst5A7exVoWc2HBiEruZ21BGanZ8T7GqsZTmrUbm2mM7pg2Ri+5xs/W64nhdwT0+f7NRvsdbk8qn2nRiQ6uut+6j6VUZm7z2t+XqeG71qY68tyriqR5crevRXjx/+HGe/J5Gfao3v0eLzZ0uXo7F0W/5/H9Yyxrq6fVSK1rxk3p94Yv59/LD8atq3sgP6j2aLONFevPy7evq4dVHD/fig0qWUoSdYTMvulrms7lek9Z7XvXz2EHOWrd6jzxWunx8NWo+nX94+/bj+zf3SVSKPdyKj20cZblmU9EXxxvdwV0wVd/jdeYcH++n1frbco42dFxq282d47qCJztHN28cLBvHJOK/CYW0G+fYrOlVGaNzNEbnuFnL0TneK2V0jhtacUjO8VY5hbjeT4BXqVMb2W9ZRkei72Y3bazsm8K6Wqusu431KcV0NSkpOhurUsvp6rqV+rSxttsldfV9k9ZTa1W2Kq2rd5VstbG2soyuDpl/trGCvMBjAiJPiCTdVa4JznuieLV+Fedt9k5KyYet8Dcx2ZWnlC0tPntE7HZTzUV01ifcJYFrA4c4JsCMQkA5gQBDhBw/9ELPp/cK1AR3N0mMk3QmpsNvPDAWKYuzKE9MlMk8z5rrUVX6OGWeGvNd133PY12OhSCJWh0ACE4QRtYxtQjtpacfFjiMLr2v1D1W4IWctbkVvS/6SRDBmGzmc/LZ46rkfWubNdYo1Ua7Q+6kRUTAyumX5Z5YeOLQ2uuUar1d2VtVxkjulLr2ntzlk2cRLaY1erwo9pqL1wmfG5/E31dRssyM8hMWB8aPLFpcr0VvqPjF8UOi64MATel6Xdw+GNDV3i0g0EndGijQKdSWF8nrrw0ONNo0HCPNQIKuypaAgq767sGCTvJ2uromcFCVfkzKXQ4JCoWPvdz0yTcobCFYfSBMIQilKTyqM8XKHDxFmU3WWo6x/xRGSr4ZSxeRP+WlNdUOv7VzrLTfgAXUJ24EF85DyGv98k8efE78yC0sZP1Y8cHDITJ28xKiKijENAG4ypvXeqWi7cpmK9zOJ0FzswUT6M74MZomWRJGbNWStwveg02/QZ0b4kgPLKy0QLIRmRC3hUyDgZJsFmJKUWCKNvEZwBxx4LkhBizgvsUwZJ6D+qBejfUYBiNrbyx0QbIhtY6TX8MkDa69Y+oSYbKoi7Fr2hOIrAl2+hgBzRUZxhBoh5Q70CVmLVJeluyGlJvSgkpSbp1YdGJTuytS3sQ+qzJGUq7UNZJyRbmi2JkvQGAWJdPk6ua5ceYvZzyOslnByS+TKU/z2T8S8y2ACJ3UrQEJnUIjMe8ZXOjEbg9g6DTah6ERZV7c+ti41fp9jYGHhbbW13OWLkCyd3EbrvhLwEt3mYdtFmtveVTHCjeI3XhJ8hn412wu3rKHwE0VgdEHbloIplxec+NjbpSMd6Id5UsaSWi855EXR78NNKxiTxDCux1W2QaVxrtCpe9XZBhUur0h0GFkjYQEObYVCGZOCMA+poBRV9Ann9jc9JBneo/am9h0MDTWYxhjob2wCqJ1wyqyZD9hlfZ2ebVgn1UZY1hFqWsMqyjKFcXybIY8hvKJs2yMn2hq79ZV6KRuzV3oFNoHkryt+EkjRKkTuz1UqdNoH4bGGD+pJ/kQ4id5bkseLklzx3hUx+COoZJ9DpVYE9dpYTvvvoVKGhm1Lpxac0O3fXrc3hDoMFTiYQpDy+bC8PkuwA7zAfMCKP5g3KLQtIQp7GMwNNZjGGOhxVAJqR0qIX2FSmBnoZIG9lmVMYZKlLrGUImiXFHs5+s8g3Z+LSDR1BAIJ19by2Mnp9epPGyMxcabxF+kUcyzMZCyBUeik7o1Z6JTaB/Y8rYCKb2NjG4xp06jfRga3QdS7n21MZKyi5GUueo3wWLlN/PQil/6TRCUfvOojj0e4yx7HGeB9sQ03d2OszQ0cB1Gibdj17ZPltvr0w4DJ3YYEu4jBGzumQBbPgMUcRvIDZLYtSkitJe9O431GMZYaCNwYkOLEGLVyDGpSrYfOEFE9AmA+bn/lnliOhMLd7Z1p4nBVWWMgROlrjFwoihXD6SMgZLeHYdO6tach06hkQ13BRh11e8IGd7HszQE9wVFMAlcrwyh4PtgWhjCozoTrAHzlTcGCj8upe0P8b3jT7JhUl3kTqhJd5vqdkiLsBtyxjkBbugTMeCxAxhiDgg8aBNomQ5Ffh+0qLEeh0OLRCM4FqHmw7SoKtkBLbpzV6w1cezuUu8bTE5VxkiLlLpGWqQot6ZFSVqsH/+csl+jzIjikhT96+//nZVLzKvl5mR5db0YydIW3IlO6tZcik6hkSxputYCNpI4Gjs2sPoaApaQmYu8X+K4bPggWRLmMV8mnOfmUWCGkipl5TLiakkxt41Hdebc7q8cuv0QqJWjMaKZ8T7yrxfcCHi6djk8NobJqCCdELPhPbZ7wKjcIKDIEraOhQKZY8hdQLBcSzdxACFxfOw1sn+PtX2N9RgGo2pvLHS0kExcRBxs9reQvEngMPq0RZZs1WbJ1uBZchODq8oYWbJS18iSFeVu4ZSR+G7B6+ukbs3z6xQaiW9XaEBXffeIQCd571cJy+XAOjOpIZ3tgcZWqQCPOky/8poz/WH6j6C/u8ZPkT3BpnWw/NR3oMUsh4DAtMSY9m0XMM/3ADUDC5nchtz2+iAtjfU4HC6DTSwaiNRIhKxKdpIICa0yEZKeYDShLurwsK1HT05VxshllLpGLqMoVxRDLnKMf/39f99a5ZODzJA+cqQ4W3AcOqlbcx46hfYe9srJAaoFntz2ymlxVKe5d39Zpw4ebm9ZR1gWY8oXvy14JoyNx+PsP9l1OswlHUQnFMGDhcxW4KPADCEgLKQAMxICBj0b+Jy7DAUoYAHvAzI31uOAIDPC1HbrJMlVJbuBzGYOmS05fTCeUKu7a38aTE5VxgiZlbpGyKwoVxR7mwTPjVM5ZAWOyIrbfj7wREyPRSIvNFBy5kb8vAUvopO6NU+iU2hcItAvEViYuMjGFuxzI9EDQsf0uM106SoJgF/axTxLLlbMopIrd1RnujXgT34ShzzlsTBaczbfJw714+//TI1lnPHUeDv9/Z/+ZyORuXEBy8rfRXOLVx8qnyITbDu7zad4yyTKBJk5ETPAFugYOsQ03T4oU02pwyBI7XVuh2SZUmSGtu8AAnkobyBngHHXAq5j84C7GDu8n6S4pnoc2ljo5MBi6Mqj2oAFLESBRYgFAf5fvfT7Y8UPo7tbTI3EtVMj8eBTI5s4WlXGGBtR6hpjI4pydw6kLTYRXviRRONjROSh2ruFCjqpW4MLOoX2ISLSjCfoqmyTK+hk9MQXdOL3odM7O3i4PdyoE9Y3dtTpsQ/DoLNoqGj6vOV7u7HrYaGDjIZqFNrN7jYp6fNM6Urq/vR3j9Hv9ZnSxQbxrIC+atS7DqQas4bKY6S5QNrxgseSs59lGbvJjCQ2FtfcUEjG/2Cz+f+UP2uqkYTG28mfJyXfGO6ecZvu+B3oXd7q5EBqQlE5h1Aem0AhYIE8RcElPHA9j/rwfvvR9q1OTfUYRhCtvbHQ1eIIpg41TYSsXs6iril1GJ3bRoTUcS3XwqRGhLQq2c2VXQggO99w4Z5Ac+LCLjePP9ryqjLGCKlS1xghVZSrTp5+J6xiKlARv0pSPhOz3TfO4zCKo0WVZDTGR/uDBTqpW4MGOoX2IUbSZXz0aXBBJ6MnyKATv/fbbOQZ1NffmEQBFgqTeFRnqjXgzSUlZKmQNuU9UOcqrfAxG9BRFWudtbIBvWqHOxt1zs5ffzj/m/Hp7D/Oz/48TOKMrIlpkd0mzh0kmGDLxDJKLw9ncB0To/tjeO2nltQUPAzK1F4Xd7n5igXIczwTWCGyAQ7l3kMm/oAB90Lf8jyb9pJJ2FiPQxsLHcZGIKYU475jI5ukDqNzW8wec2pnjzl9ZY/hDq8zf7RzVWWMsRGlrjE2oih3a6e40Kw4aVku5mRLL1swubIjE8quo8yY8QWbX99kkZ/JAnJFKIp9JqaxNA1jAGUL2EEndWv4QafQGEDpElPoZPSEK3Ti96HTO0owa4c36sT0xx11GuxD13eVZSSa3XUwMc0+T+F8QOggk4w0Cu1adyOLuBSJf/XZ3ZuFDrK7e4yM37pbJL2RNzOu8XCeZSbwMFDwsCwgo+kKHj6qg7N2P+usWegc1gqdq88F7OYlQuun5K+dhdsT/3PBaspctSQ2LsruHWj4HU9s/Mh7u/sOv3cXjzMxQaJFet3I/YDUYcTj2uvcTjbvEngMBT0hLiETYYvwhLpYtHhPSyyPk39oHd7hSktgQWIG1AfQCgOAWWgCYtsOsDnktmk5iONG+zEeOwYa6zGMsdBioqJdO1HR7itRsbtj7hq4WlXGGIxX6hqD8YpyVaLiax7zMFoYi0TddLEKul8slkH+y/kiM95F2UIwlDH6vgV/opO6NZ+iU2gfonHdRt+fwiB0MnpiETrx+9DpnW3vbpFZ6KT1zi50ihxEHqtXeEywSICyIXQVlsukx5S/RItMhu6kxzyqY4n3Jru1hbDZ6zRaRNm18cfiUyNM0hyOrBCIBCQKVhlmGA06EwfTgw2j2eL/gqX1HEbbJHUYTLq9zu0gjGYTFx17zGfBcXEbn2s6NhYjHUJL/MBeshgb63FoA6DDsBqyqR8izwLYC6BweL4NPIwpgL5tu45rIdPtJ7TaVI9hjIUWc1xJ7RxX0leOa2dhtSauV5UxhtWUusawmqLc3RzXN4m/SKM4P7LkMpnydMxf1dferevQSd2a+9AptA/BlI4jaE8gDzoZPREInfh96PSOImidkAqd1K0RC51C+zAwOrxjHrkOxHZvsfSHhQ4y3VGj0O5192q69dvdm4QOsru3kd0arJBwHidfI+GjOghr9zNXq2XUR2WuVjmos/qZq1aFOLvMXP0gr3hiU+M9+5rnr1bsRZ70LoPxZyyd3hhn8TS6ul7weCbmzUAj8XjiOgcbiYfCuDkU9nKYRE2pwwi+tde5HQZimcl46GICfGz6AFvyPFrPNIEXWDRkNkOm00sqc2M9hjEWWgvEUljjGt+qZD+BWKezQGwDy6vKGAOxSl1jIFZRrih2PjV8gazSyFsuEiPg02liyASNKDF4/is3eBD9JgxAJKaNseDZIpLfJP5SwplIQB3xvfibTafMkLkHUXKVsjBiRpinH4QCXBnAiIQcliXGdLwAZ0Pt3XokndSteSWdQvsQ0ek2vvsUSKqT0RMs1Ynf+8S4aAoqUwtyUwsKUwt4/isHpakFQQRyUyu/KU0tWJlaIE0tUE0tqEwtAFKMsLTr+xbqzOBBZNbVCSG0QOs/Cd6cLZj0dj+umpXJm+3BhZ9I9CX92TBZPHImLkW7zeK7PDIwcJ3Q8RyAPUYBxqYFCEeiibDrYR+KAeb1QvEb6zEMotfeWGg7/e7Nx/OTPNebWmLizdMkPBHTe7VMRR0HQsvsOw/vCcoMYzi0wftXuxVrHDJYlexnX6Pb2b7GBrZalTHyfqWukfcryhXFflofl8KmxtkXQc8/cTllxAgeufkWQIRO6taAhE6hfeDmXaThdA4udOK3CzB0Wu3DMOkqSwNAgCgougD0eexc1fFjpkaDAM5M8ZlAeMupeJmVzzyqY4v3I0ujjcwJ/lW5zHKVC/6vv/93Zlxec4Fx/ZthBligLQ3qbgdYukuTcAmGyDZ7TpPYKHUYFLm9zu0weuYzEzkcuoAiKwSYmj6gpuUCAm3fFa6FYMfvo+cb6zGMsdBimgSsnSYBh79frYHlVWWM4RKlrjFcoihXFPs5jfJJNmMLMS2fG3Pxu4ANX7iYYlmW/518lV/IRNB8aAtQOb0xprLYOif0/Zj8sKH2bv2MTurWfI1OoX1gzt0mPzwFaOpk9AQ2deL3PvlhXhhQUBhQsLafYGU/QW4/89O8K/MJKvMpQMg6o6HOtBwzGireLI8ojNjK+ywiFrNh0msEJ9B+5KXke5S/4Fph4Pt2CJBrIYA9SAAhNACUB4zjwDO9oJdV68Z6HBgDI5TWPjGEDp6BNZmcqoyRgSl1jQxMUW6VlsdZlsTPjZR/WU30gmyx9CpPj8zWR+C94WIOjRxrC55EJ3Vr3kSn0D5wrM72kGNkuy4m1O5rCDwsdFyZ3Myu0tw4gso25kRqbRqBMI350auBNI1HdabcuFZZpoMnS/FXIDzhaTKbs1iG8haJcZZv6RaNe22cij+XAv5V560OlF7RCbIOl175mAeB6dmAUuIAHMhzFhyCADQD5NoOIybr5SzWxnocDr3CJrYcQmsscFUlO6BXBEALQJLnA9MTjCbUtTqjVw0mpypjpFdKXSO9UpQrip2J4RRNjXmSiWkylVt/DT9NopQbmbBPhig7T3kcpPyHkVltwYnopG7NkegU2vuFDLkrcwrKeSIQNSjmCZDzBFTz5KhO8+8+yrb6Qdmnog0FMYuloZlxmRX4hS8XPwwTSEM8ge6OpwF2CKSxb9PADEIByEICcEgY8GyXA8dEPseexwK/l3WKxnocDpBe5X/h2plieB8O1Hn05FRljEBaqWsE0opyt042z7PZA+Vs8+pExxNjfSeS3EnwXC5aJNPkKt+KJ5c05im7momivlHE9bIRcm/B3eikbs3l6BQaFzM2LGasz6a999XGE3F3kWI9cCIuuGU+gWI981WPtfUEK+t5VGdWjkzs8E61RXBC6eEueIRBiCEReN91uID6DkeAIscTTRRyhzNCQr/R5tTHGszGehwOTyuWMUiNG6iqkr0seDjdLXg0mJyqjJGnKXWNPE1R7rafkz7uYjmXM7P44KebTMxT4WJG1rUF56GTujUHolNo7xc6VvMhx9SZMkHArJwgR3UafgTWKzj8lmeimQVgNgKeySTVbLbMJKY+W6bJfKhbMtyJC8luQ+gObvHND9VJfg2TNLj2yiN1bGxTF03EgJpA9R7wDqHzU1QZBnpubxh0yKSga7shRCagzJKHv1g2YH4YAt8PEXNhQMIA9TEcGusxjLHQ4oqXVXvFyxr8ilcTG63KGJmUUtfIpBTlVhnS+eQW5iObjZRpC15CJ3VrnkKn0D4sVHV0g2tXgFIneJugUqfTPgyPDtcxV+3f7zLmJqHjMuYDm7Iqv3hUx96OkZJV1OPymhsfc+NjvGNxIF8yz6ARH7+L5L0nN/LXs8V15GfDDJtAPLGtHQ+bdMiXOSEBdW0CsOVyORsY8JDLACecOBiRkLFe+HJjPYbBl9sbC52E0JB7fAGh5VALOghJCk5pH71eCs9qCT+0rv6aZMu2w2OioSF1TWLLdoZ99HEdkcPo2RaDXnbtoJc9+KBXEw+ryhiDXkpdY9BLUW51IKiag1gkyilZ3NXh5hK0XqZRPB5Jo6u9W0iok7o1WKhTaB+iH42xg67W1vCDTkAfGEInex96vLNwaFvkQCeqX4Kg02Lvk4hup+rLVCI1XZ/nbhIkcX440aJwk0d1zG/TWJm8sIV/7SFUViWc6UNl6nPiR6AxVPmV4oOuQmwXpx8vL88v3hl//PjLpw+v3hsffzQu3519fP/x7V8HGlJzJwQe7uW2HsWMmqYHiON7gtA5BBDuiibybSd0A2iZ8H6BLRPvxnoMg423NxYarhRtXCUCUDS7TV1gIxs02/HWZJkolylEjrvp8+BJjWtqq5L9hFk6u6a2idVVZYxhFqWuMcyiKFcUe7e8Soy3abKIltlzGWIxLtPl4rpcDjy9TqNMnmhvfOLT6CpSVB6DLP2BBJ3UrQEFnUL7QLk7vJX0qeBBJ6N7AKGTvPcE+1oYSHBVGMgVixb2Md8wL37xS/soXzG3j0d15l0Dds0DeQ0qkCR7R9h1Cyz57l74W3vejdOpvLpmsCkozgRZh3tIme2YNrSIBagnz8AmkAHKbAZCx+GhQ0PbJr2c9ttYj4OjVW5tWuUOn1Y1mJyqjJFWKXWNtEpRrsqwPE3iUNgQMUUkm/qRRYJWfZV/FDetlKe9/Ovv/+dNlPlCfmb8nP7+D3klW8yilP/r7/+3pGHVrd0j+erd4+ikbs3r6BQayZeefFmC/UBKEe2VeFVSxwT/BtwrZ1hrKyo5VyitKJBWdHX4mLy2Mr94ZWVCwVwSsZUFLVnaYmU+j+pMy93fJYD6IWfvuS9vCc0MvhC+SrzaXHin4rLQLD9iQSZhRfKfwkf9/g/Zwr//Iz/Y/u3E+POkuox5kOyNTCCiu83eml0CrDeTq2t4XUpt6riu1ewC4MeayZpSh0HH2uvcDqm5YweOQyEHISI2wCG0AHGQA1wv9E3Mqc1xoxvAH9vzjfUYxlhoj5ojVJeay5L9UHPSGTVvYHlVGSM1V+oaqbmiXFHsfGqkbC4RZ2KEPOAgZTJuz42YT405lyYqTUrwErE4GUn3FnyJTurW/IlOoX0g3c2ApK7KNsGkTkZPgFInfu9XPaMpKI0kkEZyZSOBsJGgtJFgWtnIozpzrwGzLlkjSxeRP+U9kOtqG0K3t54KrpotmABmxo95BncYMeMDT8CFn0jEI2odJnFG5oRguNvEuZstuCYRGNwl4l/K1bkd0qc6IofBlNrr2S6vr3UC4iFKQRB6qNg6QbjlAZNAEtgO9zzey5kKjfUYxlhokTXXuL62Kjlw1tzE7KoyRtas1DWyZkW5W7dunRivk+DmuXGx9AR6iX3+3PgpiVllaUeO3J/n0EndmvfQKbQPHLm7rddPxY06AX1gR53svSfIK+4LPGEOBQJZWUPhcYUxPKozyXZ+g21PRPjDMvZlavUfi0+Nj2G+C6U8h078euGLkevzgdJhZ4JNc7fpcJekift+GBIIaGC7AENsA0a4Cwj2rRCZhNGeSFNTPQ6NNEFaOwuYDp80NZicqoyRNCl1jaRJUe7bq4p/4gs2v76R+1nWWb3jsVUbau/Wgeikbs2J6BTaB+7U2Y5KaiOMEMSot6vQhNQHhI5JvTUvH55VFnGdqLs6oajOTNv9PN2eGNTqXMSbYgdlfnfwTwLRpbGxPkPxxjhCtg0Bcm34h2ESKWhPCHV3m0h1tK7oOpaLrF7XFR8SOQxa1F7PdkiRA+LCgDECOA5CgE2CgOdaDDimFWIzsE2b9nLVXWM9Dm0sdHJWO4GSb1uWS8U/TRtSy+ol+/5xwofR1S1GQ2pcCF6VHPqe6AYeVpUxRkOUusZoiKLc6kTvNFkkV4yzMeCxBTigk7o1SKBTaB8CHh0uFj+RDOgE9EEIdLL3occ7O6e7LWCoE9UvONRpsff5AvPS+R3VMapjckDh+F99+PDq/YU8O/vi9Pzsw+nZMGNXCE3MA76NjjkODSgJAbJtuTfEcQD1/BBQi5mcWgHFpJc7qxrrcWi0F+HamdO4L9pLO0sCaDA5VRkj7VXqGmmvolxR7HWUiekoBtM0yFdootmMy/l+YiRpdBXFeToAMz7zG8NPYp/PF+XJYDxfy5kVazlB4svVMS5LL+PoC08zNjWuWSokjSkE23A/Oqlbc0E6hfaBX3WWQmBhgi3HMXFvdFqeC7ZZ6JhCsJlMeZVJza88Kk0qWFlUmU7AgLCoYGVRy1PCcosKCosKSosqS68tKlhZ1KM6M3XnUxAg7YenXcx//3+LlMcsyrI8D766vZFLb/aGLxeZfz2V/g/ZLgTIseGJcRYvvv7+zzTkxt++RuJ7HhvvljMWR9lsmRm/iMJ/SuKQZ9nqmveIp8v46rnx8fPn5XQhSl+mLIikvuLfsviFf72crvNKPg+ULcKJSezdZotdXK4DBcMg0EEU9naxzlrigGlfe53aYQiAeJSHrs8AC7mwpr4bAI+GASA2YQ4lNnRpL6eBN9ZjGGOhxRCAVTsEYA0/BNDA4qoyxhCAUtcYAlCUu7N5+lVsnMcLLrTzF0tB319HyVUqwcpI4LfgPHRSt+ZAdAqNBL4z1Kirv3vkqJO890uh5dZpJg/vrowh8EpjeFRnkjXk3sPj3DtHT+kE24d7rxHlLnMC6ICAegRgaHIxLCVUDUPMTWw6HuklXbuxHsNgMu2NhY42ZTiCj1BHJkT10t11RA6jZ1vkqGZtjmoOf696A7Oryhg5qlLXyFEV5aobq1Y8FbzhmfE6yTIu1UxTns2FksUZLCNN7R0Z6KRuDR3oFNoHmtph5vYTEYNOQB+oQSd777mqXDRe8dWAZ8DLjaIAIapRPKoz2XY9o7evleLXqSCi2bVRHvcVJmme61Se95WE6q71gS7g2hPHxQfLkOVRd7ZtMtEgHAHsUHluA8bAIuJ3z6IWdFgflKmxHsPgUe2NhU4W8y3hcCB0sdvbYv5a4oA7tUVybNcmx/bwF3AbWFxVxkiOlbpGcqwod2sB10hi9eC2PJ37PPaZmKhy8p+UNzvLzz/xLyubUKZzX4hP4gUXP9fGqfhzmY7LvtvAETqpW8MSOoX2gU93tuz7NHyhq797jKGTvPdUulz2TWL1qLcif7uyoeW1zvLjdG1Cy/ztrDKhwC9M6FGdqbm7i8XVVur9XCyGE9va8dugOrpG2bEIpiZCxL7/8vdurlF+SOowaFB7ndthnMN1bYsGHgWe4whWxWxb/MsOge0ExPZd4lCrl55vrMcwxkKLlNipTYmd4VPiBpZXlTFSYqWukRIryhXFTsUv4nUyyYlfBWyWjVx2C75CJ3Vr/kKn0D5w2Q6vSX4iWNTJ6Akw6sTvQ6d3EsCAiLqmg03Y3xlumyWOO843By/8lbeT0Qsmvd1RHSu6+/vDqzhWt7dd5w/JLdyz58bPYhQkco93Hid/mwQDXeWHE9c53Dx427cR9VwIXJuL8W9yF1BumgAx1xM/RPz0crNXYz0Ojv3WPssa9XWWtQM7ZL+PnpyqjJH9KnWN7FdRbnUwZXq1LOhvmcB29qs8lCb28wO6hFM7ya+xPE3iRRSLgmxqnC3TZM6ZPA3FEwNw5Mtb8C46qVvzMDqF9n45kJUzCIgZVBzOVM4geSDTVRLkH/rVBAJ8NYFAkE+gozods/sQ3OkHgufWiM28NAqubqXbnkVX17ezUpQM3GFCc+hOEDrcBFxGPOzQEIMQ4wBg4lJAAoJkeroVBlBMDJ/2Ac0b63Fw0NytDc3dwUPzJpNTlTFCc6WuEZoryq1iTfwLZ1MuMzCn0VWeljki8SecntuRM9FJ3ZpD0Sm090g8XU0Y+Q75hHkQeNfphxF4HyLwJhMbHi7wdnyHWi5lgPk0BNgjUMwLRoDDGDM9RAI77GXnW2M9hgG82xsLHZ0NY5o2Nm35r17uNakjchg92yKlorUpFR0+pWpgdlUZI6VS6hoplaLcarXDEL/OWSw3sy2S2/eSzNeQZTLyqC1gA53UreEDnUL7kAzW4ekwT8QMOgF94Aad7L1nzgys7SJYJLdvF6ns4lGd6bbz58NUeYOdsuWf352//3jxUfx1+uq98e+/vPp0efbp/V+HSYeRObGps9t0uNnFtXpblt8Zi5zjCwgtCiyHoCNo/UEaGAR6WXUqFchqKzAMbtRel3e04VG0NzEtSrHZRzfXE3poXdthcMtCnFAmKmfEowCHpg2YFVLgcEwowQJJkEYXUz+24xvrcWhjoZvgFiKWaVIkgyq93NhTR+Qwera94JYJ6wa3ZMl+gluos1TeBiBKlTEGt5S6xuCWotzts53kZhSZyDsXbQFOk7mYooLCxOVlu3xi/Cm5SX//hyEpVxaJAT7Gu7YAF3RStwYZdAqN8a4uYYROQB9QQid7H3q8wz3OT+OHOhH9cESd9H3o8mYRIF2VHUWBdOL6jwTpNNn7aHd1b1exNUMCJX+Nk8ortDn4LGASB2uUdFTH+zYIf5exYJYuIn/KewiBVx5DHwJXnxM/ApcreWbFB12Fzi8uf3lzfnZhnH8w3p1fXH789Ffj1Yc3xjqi/lfj44/Gxen52YfTs2HG06E7sfGOnybXYQQOuj73LGQDlxEXYCaaxuMhB27IKMOMQT/oZct1Yz2GEadpbyx0FYGDxLIkhkW9nC9XR+Sh9WwXq2aWbR1fz68nwtMI1AppL3O5vuBhdHGLQVZUO8iKBh9kbeJZVRljkFWpawyyKsrdCrIKzX7iC3nHbxb52YlxvsiMj2l0FcVZHn99I098TuZyX7hxVKS2/GGMsm4BEuqkbg0W6hTahwBMl1HWp0FFnYA+4KJO9j70eFcht1YgpE5MfzBSp8GhhNgyMKu8JIjksYOFk8xjb0HlJFfpn0d1TO/ehNhaCJX98eMvnz68ei8DYpfvztbhMvFrFS0bZogM2RNquQcbIvM9HIQ8QHIOQIAh8gG1GAQuMuUZt8y0EG9iEx9rDxvrcXAsG9dm2bgvlm12eE3doyenKmNk2UpdI8tWlLvFskfGvAUPoZO6NS+hU+hQQPRRndbd/VNKzH4Q8ZvoN3lCdxolxnks3kx4VT+aTyPxERcu1rjwxcD9jRnc+JEHfKDY2JyY5uGe2E0cK/AJd4Bt+fKiPC5IostsQELMLM4s7Nu9YOPGehwcNrZqY2Nr+Ni4weRUZYzYWKlrxMaKct+m+a9OBlxmxmUy5Wk+7UfQvAXXoZO6NfehU+hQQPPq3t3V9ACL9fQ4qtPsgwgx9wSoX/109un89NUH4/TV5buP789PDc3RBsaJcScc/dCjry4uPp6ev7o8//hhoEAcT5BzuMcEUtviDvQswKlMiPZcDqgjl2wsTF2fmJ6D77cvLQPxxnocHBA3awNxc/CpYE0mpypjBOJKXSMQV5Qriv3EsiyaJcZPy6uYRScy3JQmwVKGn7jBplNmhPnJSWHEZOCpxO1Hn8Z0sA21d+tZdFK35l10Cu09Tp8VswfM8tkj/Ek1eYCcPGA9eUAQgRLVr9JE6nTJiOErIP06FZA3uzb+WHy6vutLOcS7OrHbGCgWRxPsHG7CCPNc7EMSAj+0fYBDDwMWmAHgvmdDzDAN3V422TTW4+CwuF0bi9vDD4o3mJyqjBGLK3WNWFxRbrXqy+XkkIiCceMsFh+KIZX9YLyWN/GlIZ8Geby82r3xMV4k0+Qqd32fStswQvItOBid1K05GZ1Cew/JA2USAV5OIuCt51AeVF+ndierKSRvuVy/9VGdrhkENEc1oLn63Cw/EwFViH7W6SEJ78/OX384/5vx6ew/zs/+PEzIDsmE0B0/BqGjM2YdTDFFDkZ2L6i8ptRhYPD2Orejcw0c1zYtS6L6fo6xqCHy0Hq2m9PA3fwcKIxRftMAtLH9lz46uBSe3RH+69jV3QZVuGNSQh0KPIdDgAkmgMndaTZlPnaZaYeW3ccAaKzHMMZCi0EVUjuoQvoKqlgd3pb1aPikyhiDKkpdY1BFUe52pmHAjTc8MS7TKE5OqnuyBG+ZGiybc3+RyTBKFVzxk9jnc2kZ5OdyoUE+K6bZGGDZgrPRSd2aw9EptA8HInR4BMYTWYVOQB/MQid7H3q8w4OGnxgk0MnoKVCgE78Pnd7dUcMtUUydqH5ppk6LvQ+ll/kqARf/JWAhoRO4hZzACjnJ6Pk6pF4BJ/mxPJt4UQCnozoOeW/C6i2Exz+dvT9/e/7xlwtjdZrwMCPkCE0oMnc7Qt5JqI3AY0gdSFwCEYIQ2xZyewu11RY+jPBKe13d4YV7GFsm7fvCvQ1CD61rO4yiOj4hgUNtwH3iA2wxAoiLfeCTIPQs12Uw7KXjG+sxjLHQYhTVrR1FdQcfRW3iYlUZYxRVqWuMoirKFcUur2X4NOQC2ZfR0J9uZEaNgIh3wqPrq9vOY5+JGSytwon40Dj7lc3mU347yvrdR1E0/c74xFk2bvvW1N6tB9JJ3ZoX0im0DzGXjm/0ag5AdSL6AaE66fvQ5Z2F2dqilzpR/VJMnRZ7H2aTEbKg8K1lwGxW+tY7EbT1bV9R5VvFZ4AXrvVWGC6RjlW0i/SrR3Xs9Rh5q2JqB7HXzJkgh+x2WK5DQo8RCblpM+D6rimmBAwAY9ACnHsYYur5PsF9EPrGehwcoXdqE3pn+HvNGkxOVcZI6JW6RkKvKFcR+g/8q3HKBCRJrpYVrWfxMvPTaF7kQ11eJzOWCXoejResb8Ob6KRuzaPoFDoImB7zr8AvZ8warFczpvhIzhjxplFwVKcnRtitLnifvxn4WjedOK5zsKDaswm3TA8C4nMx4G0XAmaZDPDQsSD0A9NyejnVuLEeBweqaW1QTYe/StZgcqoyRlCt1DWCakW51V6DKAj49LnxZhn7bFzM2oaj0EndmrPQKbQPKxtR5sWtjwxEbBs6LsSkt7WMzRJb6+W5wOuCIewbMZrmVg8EudE7qjOZGlCeIPIX+ZUoN/JQj/SmB84Da3Ae9blZflQGrKj1TH92hvpcwG5eomojkvy1K371S/w5Tr7Gw6RWkExM+3DXK5Bn2xblDnCwawNMbAt4zOUCO1uBh3zHplYv6xWN9Tg0amXButRKlhw4tWoyOVUZI7VS6hqplaJcUex8aqTRlMV+lBgBv5pGRrZYBpEhw6wsjgwWGK88ngacj7xrG15EJ3VrnkSn0N6vU0RTUM4UkM8UkM8UsJopgAWArWbKUZ0e2P1LGesA9RbA84/rg/C5cbqcLpYpM2I+NS74YsF9Ma4TQ6YNsTiO/GSgENueYGfHz7LrEGJbjh0wikPAbdsDmGITEO4gQDzL8gITce73chZaYz0ODmLXvgrG6u0qGNwZxG4wOVUZI8RW6hohtqJcUey789mMywn+Xb6F5zs+m/M08q+TlGdR9t3EkDlD4keel5sfm/QpSYokoXcsFdXcGFGsOat6hORb8Do6qVvzPDqFxqUQTddSl1CICbIty6R9DYGHhY4LIg8wsJUdzXdr3LaixWaOyoaCVNpQmTZ2XdhQAUQ0Z5Uf1ZmbO07WEKW0H7ImfdVfk2V8ZahXDV9HmbJv47mBHOwA5Mhz9gdJ15wJcQ+XrmGHhxRbAoC7kAPsEAdQ7CIQhsSjEHI/dHqha431GAZda28sNHSSG6ATxi7EyIK9HKvzgLRh9GaL5BvVJt9o+OS7galVZYzkW6lrJN+KcqsLyqcC893izEYSG2+T4PlqV86VAIWpGGbr4zV+isQbxHG0nI3segsgQSd1a0BBp9DIrjsBD7q6OwYQOrF7v6zJcgOpEmIgSOFVEqz2YJX2cX1EwmxtH4/qzLuGzHl4jPlJJPbWd7KKVW/f+rz6ZyXgxXHK/SQNToq/Xv7b/weirs3DHDIDAA==\"\n" + + " }"; + Map errors = Maps.newHashMap(); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + cm.setMaxTotal(nThreads); + CloseableHttpClient client = HttpClients.custom().setConnectionManager(cm).build(); + for (int i = 40000; i < 41000; i++) { + String orcidId = String.format("%s", i); + String record = String.format(recordTemplate, orcidId); + countAll++; + if(partial == nTasks) { + System.out.println("Waiting for tasks to complete before resubmitting to executor (countAll = "+countAll+") . . . "); + System.out.println("Getting replies"); + long startWait = System.currentTimeMillis(); + for(Future res : resList){ + if(res.get() == 200) countOk++; + } + resList.clear(); + partial = 0; + System.out.println(". . . Ready to submit again after "+(System.currentTimeMillis() - startWait)+" ms" ); + } + partial++; + Future res = executorService.submit( () -> { + CloseableHttpResponse responsePPOST = null; + try { + + String url = String.format(BASE_CFG_URL, indexHost, indexName, indexType, orcidId); + HttpPost post = new HttpPost(url); + post.setHeader("Accept", "application/json"); + post.setHeader("Content-type", "application/json"); + StringEntity entity = new StringEntity(record); + post.setEntity(entity); + responsePPOST = client.execute(post); + int statusCode = responsePPOST.getStatusLine().getStatusCode(); + switch (statusCode) { + case 200: + case 201: + return statusCode; + default: + System.out.println(responsePPOST.getStatusLine().getStatusCode() + ": " + responsePPOST.getStatusLine().getReasonPhrase()); + System.out.println("Source record causing error: " + record); + errors.merge(statusCode, 1, Integer::sum); + return statusCode; + } + } catch (ConnectException ce) { + throw ce; + } + catch (IOException e) { + e.printStackTrace(); + errors.merge(-1, 1, Integer::sum); + } + finally{ + if(responsePPOST != null) responsePPOST.close(); + } + return -1; + }); + resList.add(res); + } + executorService.shutdown(); + + //now let's wait for the results. We can block ourselves here: we have nothing else to do + System.out.println("Waiting for responses"); + for(Future res : resList){ + if(res.get() == 200) countOk++; + } + client.close(); + cm.shutdown(); + + System.out.println("countOk: "+countOk); + System.out.println("countAll: "+countAll); + System.out.println("errors count: "+errors.size()); + } }