dnet-hadoop/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/ElasticSearchTest.java

package eu.dnetlib.doiboost.orcid;

import java.io.IOException;
import java.net.ConnectException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.commons.io.IOUtils;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.junit.jupiter.api.Test;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

public class ElasticSearchTest {

	private static final String BASE_CFG_URL = "http://%s:9200/%s/%s/%s?pretty";

	private String indexHost;
	private String indexName;
	private String indexType;
	private String record;
	private final int readTimeout = 30000;

	private final int nThreads = 4;
	private final int nTasks = 150;
	private final ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
	private final List<Future<Integer>> resList = Lists.newArrayList();

	public void setup() {
		indexHost = "ip-90-147-167-25.ct1.garrservices.it";
		indexName = "orcid_update";
		indexType = "record";
		record = "{\n"
			+ "          \"timestamp\": 1540825815212,\n"
			+ "          \"pid\": \"0000-0002-5214-479X\",\n"
			+ "          \"blob\": \"H4sIAAAAAAAAAO19zXLcOLbm/j4Fw4spdYSRAkiQBDQuV9iyyla3y65rqW7/bCZAEpR4nUnmJTPtUq36HWZ1I2Z2s+l36F29ST/JAGQyCclCiqJIZjKT3SXbygRxDvFzzvcdHAAvfvh1NjW+8DSLkvj7Z2gCnxk89pMgiq++f/bL5Y+APDOyBYsDNk1i/v2zG549++Hlv71IuZ+kwUnxlzFni+vvnx1D8T8gfkxgmwgD7NK/PDOEgDg7ieIFT2M2/f7Z9WIxPzk+/vr16yRJ/SgQf14dx9lxWaJ8ggdLny1ytXSPrIuUzwRRtohif/NTSqH1c3wu3oQteLDhsXWZ8qlkcc1TELMZ1z9VlSmfmvGZJ9r7Oprrn6rKrFsjTZN0Q0vIr8uyfjKbbWqA4vuydNGF+tLF92XpuVArEb0EAr5g0TTTP3e3ZFnDZ37zdaPAVYH1m8/E0xveXH69LvtrMYZAFPB4EYUR39Rm3xYu6wmXcTEDdM+uCqxbJeUhT8XE4ZsapCpUPseCIOXZhmdWBapZ9CUSAxDMkyzaPMjvlixrEC37Wf+U/LYseS2mSZLe6AuvClT9NJ8mNzPRlJs6qyxTPvVfSzYVTf/QTL9VrHw24+mXyN8w+VYFbo/dh0bsum+EhRBtGG3q0qpMNZsyzlJfTvplumHY3i5XaSh+T/mXiH/dpOa6UPmct5xu6FX57V39RAVZskw3td43RZ+9/DdD/O9FYT9O8sLK5Cm+VUss0+ilrD0T1VdV3+MlXhwrT3xTjfQuLzc9lRf45rHrJFu8XItdF84/Ll7keOObvFAmrDrDFUlqiWnisyl/yeMXx/d8vBK4scoXqxlVTj1FUPmNn/J8CoAZX1wnwcs3kTDPixfHuu+/rSFberMok+4eBMKZvTQhcgG0ATIvITnB9olNJo5j/q2q8+4T3zT0lGULMBOYQbRdUNZqQlkrdC8hPBH/mXCCkPO3dZPf88w9bztl0YwHLxfpkivvuPr02/ICxxT15U7hzmN3vtzw9DyNZiy92VzL7UKr7r23A18UdmVlfjZgpdIA1WxhRAESVdBL0zzB5AS6E4pRzRZeqSRxifElyiIvmkYLYevnS28a+c9WSt6H56pKVAXzcaeophlS95W+t777X7isFYkhZZ6YeGLDmkNKeekKkZxcRV94nIOz7OVPLI2Y8SnJ2ItvoMutgg/UGbJZNL3Jy758FS+Sq5j99tt9daoFH6hTtFcQLYqilZ7GxurVZ6p+P1Y6Xvm4QqkKqM02DVWl2LNjpabbfu2OO9xY452imoFWcyY42HzEwNiotDFfLoBgRMJTIoIdE5nPNkyZOq92vK5HUJH5lN2AKA74r2K+3XnpTTNs8yvrZ1jXjanWX8CGb7//tgzwp5HwvsIH319afeJ+VFFUcPzq55+B/Zd3/355Zlo//+J+fHVxSe9HF7rqczCxsZrbcENXz4Pw45sHjx/VJHcaMJ/RfxKc5LvMOE2mU37FjfdJHCTx3Xq/tTj3SL+nwJ1ZIn42CDXmaRJGUy6mxZVwn7pn60hZd/ZnfzpfpnzJFhPxzwnzJ8vPx/MkXbDpMY9Xpk2QZ2khQSosJGBrC3kUEjPwCA/FkBXQBHs+BAyGCPguDngQWA7x0R8m14vZ9D5t7xiMuyVu/6ra2832UCmZ44iC7G40kwXfVU3uiiOWZHKz3S4JpVrBinCXzHzj82WZWxXcw6Tvo+Kb3+ue8k/zAbXR0GPeQfEG1LRsCh/tDO570eOyric4BAIgAojI97cF6iITZJGOHMKjWlatf3QI++QQVt8rAxosbub85YWfzJeZ8WopKGhqnL9Z1/RNyVp1fmHTJX9p26ZJHYRdCO+tryhVq8LSp6yCHlmu7kQUPGZC5Sg4LjD0RJT7geUvcR58/5D8b5yETnrKpzlFl9HdlxmfhvdWd6vUHedT01SpbqiuhV7HKVTKuvqwCnYpsTGQLWeSB2/0OVWUrHHwwLZJTWqrqKlE+jf6HrXcbdda1bVea9jsxdaltPWsQ6APuPl1scd6QVOY69wL0BNkTdxHMSG1j0OB36IimnSVJst5A7exVoWc2HBiEruZ21BGanZ8T7GqsZTmrUbm2mM7pg2Ri+5xs/W64nhdwT0+f7NRvsdbk8qn2nRiQ6uut+6j6VUZm7z2t+XqeG71qY68tyriqR5crevRXjx/+HGe/J5Gfao3v0eLzZ0uXo7F0W/5/H9Yyxrq6fVSK1rxk3p94Yv59/LD8atq3sgP6j2aLONFevPy7evq4dVHD/fig0qWUoSdYTMvulrms7lek9Z7XvXz2EHOWrd6jzxWunx8NWo+nX94+/bj+zf3SVSKPdyKj20cZblmU9EXxxvdwV0wVd/jdeYcH++n1frbco42dFxq282d47qCJztHN28cLBvHJOK/CYW0G+fYrOlVGaNzNEbnuFnL0TneK2V0jhtacUjO8VY5hbjeT4BXqVMb2W9ZRkei72Y3bazsm8K6Wqusu431KcV0NSkpOhurUsvp6rqV+rSxttsldfV9k9ZTa1W2Kq2rd5VstbG2soyuDpl/trGCvMBjAiJPiCTdVa4JznuieLV+Fedt9k5KyYet8Dcx2ZWnlC0tPntE7HZTzUV01ifcJYFrA4c4JsCMQkA5gQBDhBw/9ELPp/cK1AR3N0mMk3QmpsNvPDAWKYuzKE9MlMk8z5rrUVX6OGWeGvNd133PY12OhSCJWh0ACE4QRtYxtQjtpacfFjiMLr2v1D1W4IWctbkVvS/6SRDBmGzmc/LZ46rkfWubNdYo1Ua7Q+6kRUTAyumX5Z5YeOLQ2uuUar1d2VtVxkjulLr2ntzlk2cRLaY1erwo9pqL1wmfG5/E31dRssyM8hMWB8aPLFpcr0VvqPjF8UOi64MATel6Xdw+GNDV3i0g0EndGijQKdSWF8nrrw0ONNo0HCPNQIKuypaAgq767sGCTvJ2uromcFCVfkzKXQ4JCoWPvdz0yTcobCFYfSBMIQilKTyqM8XKHDxFmU3WWo6x/xRGSr4ZSxeRP+WlNdUOv7VzrLTfgAXUJ24EF85DyGv98k8efE78yC0sZP1Y8cHDITJ28xKiKijENAG4ypvXeqWi7cpmK9zOJ0FzswUT6M74MZomWRJGbNWStwveg02/QZ0b4kgPLKy0QLIRmRC3hUyDgZJsFmJKUWCKNvEZwBxx4LkhBizgvsUwZJ6D+qBejfUYBiNrbyx0QbIhtY6TX8MkDa69Y+oSYbKoi7Fr2hOIrAl2+hgBzRUZxhBoh5Q70CVmLVJeluyGlJvSgkpSbp1YdGJTuytS3sQ+qzJGUq7UNZJyRbmi2JkvQGAWJdPk6ua5ceYvZzyOslnByS+TKU/z2T8S8y2ACJ3UrQEJnUIjMe8ZXOjEbg9g6DTah6ERZV7c+ti41fp9jYGHhbbW13OWLkCyd3EbrvhLwEt3mYdtFmtveVTHCjeI3XhJ8hn412wu3rKHwE0VgdEHbloIplxec+NjbpSMd6Id5UsaSWi855EXR78NNKxiTxDCux1W2QaVxrtCpe9XZBhUur0h0GFkjYQEObYVCGZOCMA+poBRV9Ann9jc9JBneo/am9h0MDTWYxhjob2wCqJ1wyqyZD9hlfZ2ebVgn1UZY1hFqWsMqyjKFcXybIY8hvKJs2yMn2hq79ZV6KRuzV3oFNoHkryt+EkjRKkTuz1UqdNoH4bGGD+pJ/kQ4id5bkseLklzx3hUx+COoZJ9DpVYE9dpYTvvvoVKGhm1Lpxac0O3fXrc3hDoMFTiYQpDy+bC8PkuwA7zAfMCKP5g3KLQtIQp7GMwNNZjGGOhxVAJqR0qIX2FSmBnoZIG9lmVMYZKlLrGUImiXFHs5+s8g3Z+LSDR1BAIJ19by2Mnp9epPGyMxcabxF+kUcyzMZCyBUeik7o1Z6JTaB/Y8rYCKb2NjG4xp06jfRga3QdS7n21MZKyi5GUueo3wWLlN/PQil/6TRCUfvOojj0e4yx7HGeB9sQ03d2OszQ0cB1Gibdj17ZPltvr0w4DJ3YYEu4jBGzumQBbPgMUcRvIDZLYtSkitJe9O431GMZ
			+ "        }";
	}

	// @Test
	public void testSingleFeed() throws Exception {
		setup();
		String url = String.format(BASE_CFG_URL, indexHost, indexName, indexType, "0000-0002-5214-479X");
		System.out.println("Sending to: " + url);
		System.out.println("record: " + record);

		System.out.println("response from ES: " + getResponse(url, record));
	}

//  @Test
	public void testMultipleFeed() throws Exception {
		setup();
		String recordTemplate = "{\n"
			+ "          \"timestamp\": 1540825815212,\n"
			+ "          \"pid\": \"0000-0000-0000-%s\",\n"
			+ "          \"blob\": \"H4sIAAAAAAAAAO19zXLcOLbm/j4Fw4spdYSRAkiQBDQuV9iyyla3y65rqW7/bCZAEpR4nUnmJTPtUq36HWZ1I2Z2s+l36F29ST/JAGQyCclCiqJIZjKT3SXbygRxDvFzzvcdHAAvfvh1NjW+8DSLkvj7Z2gCnxk89pMgiq++f/bL5Y+APDOyBYsDNk1i/v2zG549++Hlv71IuZ+kwUnxlzFni+vvnx1D8T8gfkxgmwgD7NK/PDOEgDg7ieIFT2M2/f7Z9WIxPzk+/vr16yRJ/SgQf14dx9lxWaJ8ggdLny1ytXSPrIuUzwRRtohif/NTSqH1c3wu3oQteLDhsXWZ8qlkcc1TELMZ1z9VlSmfmvGZJ9r7Oprrn6rKrFsjTZN0Q0vIr8uyfjKbbWqA4vuydNGF+tLF92XpuVArEb0EAr5g0TTTP3e3ZFnDZ37zdaPAVYH1m8/E0xveXH69LvtrMYZAFPB4EYUR39Rm3xYu6wmXcTEDdM+uCqxbJeUhT8XE4ZsapCpUPseCIOXZhmdWBapZ9CUSAxDMkyzaPMjvlixrEC37Wf+U/LYseS2mSZLe6AuvClT9NJ8mNzPRlJs6qyxTPvVfSzYVTf/QTL9VrHw24+mXyN8w+VYFbo/dh0bsum+EhRBtGG3q0qpMNZsyzlJfTvplumHY3i5XaSh+T/mXiH/dpOa6UPmct5xu6FX57V39RAVZskw3td43RZ+9/DdD/O9FYT9O8sLK5Cm+VUss0+ilrD0T1VdV3+MlXhwrT3xTjfQuLzc9lRf45rHrJFu8XItdF84/Ll7keOObvFAmrDrDFUlqiWnisyl/yeMXx/d8vBK4scoXqxlVTj1FUPmNn/J8CoAZX1wnwcs3kTDPixfHuu+/rSFberMok+4eBMKZvTQhcgG0ATIvITnB9olNJo5j/q2q8+4T3zT0lGULMBOYQbRdUNZqQlkrdC8hPBH/mXCCkPO3dZPf88w9bztl0YwHLxfpkivvuPr02/ICxxT15U7hzmN3vtzw9DyNZiy92VzL7UKr7r23A18UdmVlfjZgpdIA1WxhRAESVdBL0zzB5AS6E4pRzRZeqSRxifElyiIvmkYLYevnS28a+c9WSt6H56pKVAXzcaeophlS95W+t777X7isFYkhZZ6YeGLDmkNKeekKkZxcRV94nIOz7OVPLI2Y8SnJ2ItvoMutgg/UGbJZNL3Jy758FS+Sq5j99tt9daoFH6hTtFcQLYqilZ7GxurVZ6p+P1Y6Xvm4QqkKqM02DVWl2LNjpabbfu2OO9xY452imoFWcyY42HzEwNiotDFfLoBgRMJTIoIdE5nPNkyZOq92vK5HUJH5lN2AKA74r2K+3XnpTTNs8yvrZ1jXjanWX8CGb7//tgzwp5HwvsIH319afeJ+VFFUcPzq55+B/Zd3/355Zlo//+J+fHVxSe9HF7rqczCxsZrbcENXz4Pw45sHjx/VJHcaMJ/RfxKc5LvMOE2mU37FjfdJHCTx3Xq/tTj3SL+nwJ1ZIn42CDXmaRJGUy6mxZVwn7pn60hZd/ZnfzpfpnzJFhPxzwnzJ8vPx/MkXbDpMY9Xpk2QZ2khQSosJGBrC3kUEjPwCA/FkBXQBHs+BAyGCPguDngQWA7x0R8m14vZ9D5t7xiMuyVu/6ra2832UCmZ44iC7G40kwXfVU3uiiOWZHKz3S4JpVrBinCXzHzj82WZWxXcw6Tvo+Kb3+ue8k/zAbXR0GPeQfEG1LRsCh/tDO570eOyric4BAIgAojI97cF6iITZJGOHMKjWlatf3QI++QQVt8rAxosbub85YWfzJeZ8WopKGhqnL9Z1/RNyVp1fmHTJX9p26ZJHYRdCO+tryhVq8LSp6yCHlmu7kQUPGZC5Sg4LjD0RJT7geUvcR58/5D8b5yETnrKpzlFl9HdlxmfhvdWd6vUHedT01SpbqiuhV7HKVTKuvqwCnYpsTGQLWeSB2/0OVWUrHHwwLZJTWqrqKlE+jf6HrXcbdda1bVea9jsxdaltPWsQ6APuPl1scd6QVOY69wL0BNkTdxHMSG1j0OB36IimnSVJst5A7exVoWc2HBiEruZ21BGanZ8T7GqsZTmrUbm2mM7pg2Ri+5xs/W64nhdwT0+f7NRvsdbk8qn2nRiQ6uut+6j6VUZm7z2t+XqeG71qY68tyriqR5crevRXjx/+HGe/J5Gfao3v0eLzZ0uXo7F0W/5/H9Yyxrq6fVSK1rxk3p94Yv59/LD8atq3sgP6j2aLONFevPy7evq4dVHD/fig0qWUoSdYTMvulrms7lek9Z7XvXz2EHOWrd6jzxWunx8NWo+nX94+/bj+zf3SVSKPdyKj20cZblmU9EXxxvdwV0wVd/jdeYcH++n1frbco42dFxq282d47qCJztHN28cLBvHJOK/CYW0G+fYrOlVGaNzNEbnuFnL0TneK2V0jhtacUjO8VY5hbjeT4BXqVMb2W9ZRkei72Y3bazsm8K6Wqusu431KcV0NSkpOhurUsvp6rqV+rSxttsldfV9k9ZTa1W2Kq2rd5VstbG2soyuDpl/trGCvMBjAiJPiCTdVa4JznuieLV+Fedt9k5KyYet8Dcx2ZWnlC0tPntE7HZTzUV01ifcJYFrA4c4JsCMQkA5gQBDhBw/9ELPp/cK1AR3N0mMk3QmpsNvPDAWKYuzKE9MlMk8z5rrUVX6OGWeGvNd133PY12OhSCJWh0ACE4QRtYxtQjtpacfFjiMLr2v1D1W4IWctbkVvS/6SRDBmGzmc/LZ46rkfWubNdYo1Ua7Q+6kRUTAyumX5Z5YeOLQ2uuUar1d2VtVxkjulLr2ntzlk2cRLaY1erwo9pqL1wmfG5/E31dRssyM8hMWB8aPLFpcr0VvqPjF8UOi64MATel6Xdw+GNDV3i0g0EndGijQKdSWF8nrrw0ONNo0HCPNQIKuypaAgq767sGCTvJ2uromcFCVfkzKXQ4JCoWPvdz0yTcobCFYfSBMIQilKTyqM8XKHDxFmU3WWo6x/xRGSr4ZSxeRP+WlNdUOv7VzrLTfgAXUJ24EF85DyGv98k8efE78yC0sZP1Y8cHDITJ28xKiKijENAG4ypvXeqWi7cpmK9zOJ0FzswUT6M74MZomWRJGbNWStwveg02/QZ0b4kgPLKy0QLIRmRC3hUyDgZJsFmJKUWCKNvEZwBxx4LkhBizgvsUwZJ6D+qBejfUYBiNrbyx0QbIhtY6TX8MkDa69Y+oSYbKoi7Fr2hOIrAl2+hgBzRUZxhBoh5Q70CVmLVJeluyGlJvSgkpSbp1YdGJTuytS3sQ+qzJGUq7UNZJyRbmi2JkvQGAWJdPk6ua5ceYvZzyOslnByS+TKU/z2T8S8y2ACJ3UrQEJnUIjMe8ZXOjEbg9g6DTah6ERZV7c+ti41fp9jYGHhbbW13OWLkCyd3EbrvhLwEt3mYdtFmtveVTHCjeI3XhJ8hn412wu3rKHwE0VgdEHbloIplxec+NjbpSMd6Id5UsaSWi855EXR78NNKxiTxDCux1W2QaVxrtCpe9XZBhUur0h0GFkjYQEObYVCGZOCMA+poBRV9Ann9jc9JBneo/am9h0MDTWYxhjob2wCqJ1wyqyZD9hlfZ2ebVgn1UZY1hFqWsMqyjKFcXybIY8hvKJs2yMn2hq79ZV6KRuzV3oFNoHkryt+EkjRKkTuz1UqdNoH4bGGD+pJ/kQ4id5bkseLklzx3hUx+COoZJ9DpVYE9dpYTvvvoVKGhm1Lpxac0O3fXrc3hDoMFTiYQpDy+bC8PkuwA7zAfMCKP5g3KLQtIQp7GMwNNZjGGOhxVAJqR0qIX2FSmBnoZIG9lmVMYZKlLrGUImiXFHs5+s8g3Z+LSDR1BAIJ19by2Mnp9epPGyMxcabxF+kUcyzMZCyBUeik7o1Z6JTaB/Y8rYCKb2NjG4xp06jfRga3QdS7n21MZKyi5GUueo3wWLlN/PQil/6TRCUfvOojj0e4yx7HGeB9sQ03d2OszQ0cB1Gibdj17ZPltvr0w4DJ3YYEu4jBGzumQBbPgMUcRvIDZLYtSkitJe9O431GMZ
			+ "        }";
		for (int i = 0; i < 10000; i++) {
			String orcidId = String.format("0000-0000-0000-%s", i);
			String url = String.format(BASE_CFG_URL, indexHost, indexName, indexType, orcidId);
			String recordTs = String.format(recordTemplate, i);
			getResponse(url, recordTs);
		}
		System.out.println();
	}

	private String getResponse(final String url, final String json) {
		CloseableHttpClient client = HttpClients.createDefault();
		try {

			HttpPost httpPost = new HttpPost(url);
			if (json != null) {
				StringEntity entity = new StringEntity(json);
				httpPost.setEntity(entity);
				httpPost.setHeader("Accept", "application/json");
				httpPost.setHeader("Content-type", "application/json");
			}
			CloseableHttpResponse response = client.execute(httpPost);

			return IOUtils.toString(response.getEntity().getContent());
		} catch (Throwable e) {
			throw new RuntimeException("Error on executing request ", e);
		} finally {
			try {
				client.close();
			} catch (IOException e) {
				throw new RuntimeException("Unable to close client ", e);
			}
		}
	}

//	@Test
	public void testMultiThreadFeed() throws Exception {
		setup();
		int countAll = 0;
		int countOk = 0;
		int partial = 0;
		String recordTemplate = "{\n"
			+ "          \"timestamp\": 1540825815212,\n"
			+ "          \"pid\": \"%s\",\n"
			+ "          \"blob\": \"H4sIAAAAAAAAAO19zXLcOLbm/j4Fw4spdYSRAkiQBDQuV9iyyla3y65rqW7/bCZAEpR4nUnmJTPtUq36HWZ1I2Z2s+l36F29ST/JAGQyCclCiqJIZjKT3SXbygRxDvFzzvcdHAAvfvh1NjW+8DSLkvj7Z2gCnxk89pMgiq++f/bL5Y+APDOyBYsDNk1i/v2zG549++Hlv71IuZ+kwUnxlzFni+vvnx1D8T8gfkxgmwgD7NK/PDOEgDg7ieIFT2M2/f7Z9WIxPzk+/vr16yRJ/SgQf14dx9lxWaJ8ggdLny1ytXSPrIuUzwRRtohif/NTSqH1c3wu3oQteLDhsXWZ8qlkcc1TELMZ1z9VlSmfmvGZJ9r7Oprrn6rKrFsjTZN0Q0vIr8uyfjKbbWqA4vuydNGF+tLF92XpuVArEb0EAr5g0TTTP3e3ZFnDZ37zdaPAVYH1m8/E0xveXH69LvtrMYZAFPB4EYUR39Rm3xYu6wmXcTEDdM+uCqxbJeUhT8XE4ZsapCpUPseCIOXZhmdWBapZ9CUSAxDMkyzaPMjvlixrEC37Wf+U/LYseS2mSZLe6AuvClT9NJ8mNzPRlJs6qyxTPvVfSzYVTf/QTL9VrHw24+mXyN8w+VYFbo/dh0bsum+EhRBtGG3q0qpMNZsyzlJfTvplumHY3i5XaSh+T/mXiH/dpOa6UPmct5xu6FX57V39RAVZskw3td43RZ+9/DdD/O9FYT9O8sLK5Cm+VUss0+ilrD0T1VdV3+MlXhwrT3xTjfQuLzc9lRf45rHrJFu8XItdF84/Ll7keOObvFAmrDrDFUlqiWnisyl/yeMXx/d8vBK4scoXqxlVTj1FUPmNn/J8CoAZX1wnwcs3kTDPixfHuu+/rSFberMok+4eBMKZvTQhcgG0ATIvITnB9olNJo5j/q2q8+4T3zT0lGULMBOYQbRdUNZqQlkrdC8hPBH/mXCCkPO3dZPf88w9bztl0YwHLxfpkivvuPr02/ICxxT15U7hzmN3vtzw9DyNZiy92VzL7UKr7r23A18UdmVlfjZgpdIA1WxhRAESVdBL0zzB5AS6E4pRzRZeqSRxifElyiIvmkYLYevnS28a+c9WSt6H56pKVAXzcaeophlS95W+t777X7isFYkhZZ6YeGLDmkNKeekKkZxcRV94nIOz7OVPLI2Y8SnJ2ItvoMutgg/UGbJZNL3Jy758FS+Sq5j99tt9daoFH6hTtFcQLYqilZ7GxurVZ6p+P1Y6Xvm4QqkKqM02DVWl2LNjpabbfu2OO9xY452imoFWcyY42HzEwNiotDFfLoBgRMJTIoIdE5nPNkyZOq92vK5HUJH5lN2AKA74r2K+3XnpTTNs8yvrZ1jXjanWX8CGb7//tgzwp5HwvsIH319afeJ+VFFUcPzq55+B/Zd3/355Zlo//+J+fHVxSe9HF7rqczCxsZrbcENXz4Pw45sHjx/VJHcaMJ/RfxKc5LvMOE2mU37FjfdJHCTx3Xq/tTj3SL+nwJ1ZIn42CDXmaRJGUy6mxZVwn7pn60hZd/ZnfzpfpnzJFhPxzwnzJ8vPx/MkXbDpMY9Xpk2QZ2khQSosJGBrC3kUEjPwCA/FkBXQBHs+BAyGCPguDngQWA7x0R8m14vZ9D5t7xiMuyVu/6ra2832UCmZ44iC7G40kwXfVU3uiiOWZHKz3S4JpVrBinCXzHzj82WZWxXcw6Tvo+Kb3+ue8k/zAbXR0GPeQfEG1LRsCh/tDO570eOyric4BAIgAojI97cF6iITZJGOHMKjWlatf3QI++QQVt8rAxosbub85YWfzJeZ8WopKGhqnL9Z1/RNyVp1fmHTJX9p26ZJHYRdCO+tryhVq8LSp6yCHlmu7kQUPGZC5Sg4LjD0RJT7geUvcR58/5D8b5yETnrKpzlFl9HdlxmfhvdWd6vUHedT01SpbqiuhV7HKVTKuvqwCnYpsTGQLWeSB2/0OVWUrHHwwLZJTWqrqKlE+jf6HrXcbdda1bVea9jsxdaltPWsQ6APuPl1scd6QVOY69wL0BNkTdxHMSG1j0OB36IimnSVJst5A7exVoWc2HBiEruZ21BGanZ8T7GqsZTmrUbm2mM7pg2Ri+5xs/W64nhdwT0+f7NRvsdbk8qn2nRiQ6uut+6j6VUZm7z2t+XqeG71qY68tyriqR5crevRXjx/+HGe/J5Gfao3v0eLzZ0uXo7F0W/5/H9Yyxrq6fVSK1rxk3p94Yv59/LD8atq3sgP6j2aLONFevPy7evq4dVHD/fig0qWUoSdYTMvulrms7lek9Z7XvXz2EHOWrd6jzxWunx8NWo+nX94+/bj+zf3SVSKPdyKj20cZblmU9EXxxvdwV0wVd/jdeYcH++n1frbco42dFxq282d47qCJztHN28cLBvHJOK/CYW0G+fYrOlVGaNzNEbnuFnL0TneK2V0jhtacUjO8VY5hbjeT4BXqVMb2W9ZRkei72Y3bazsm8K6Wqusu431KcV0NSkpOhurUsvp6rqV+rSxttsldfV9k9ZTa1W2Kq2rd5VstbG2soyuDpl/trGCvMBjAiJPiCTdVa4JznuieLV+Fedt9k5KyYet8Dcx2ZWnlC0tPntE7HZTzUV01ifcJYFrA4c4JsCMQkA5gQBDhBw/9ELPp/cK1AR3N0mMk3QmpsNvPDAWKYuzKE9MlMk8z5rrUVX6OGWeGvNd133PY12OhSCJWh0ACE4QRtYxtQjtpacfFjiMLr2v1D1W4IWctbkVvS/6SRDBmGzmc/LZ46rkfWubNdYo1Ua7Q+6kRUTAyumX5Z5YeOLQ2uuUar1d2VtVxkjulLr2ntzlk2cRLaY1erwo9pqL1wmfG5/E31dRssyM8hMWB8aPLFpcr0VvqPjF8UOi64MATel6Xdw+GNDV3i0g0EndGijQKdSWF8nrrw0ONNo0HCPNQIKuypaAgq767sGCTvJ2uromcFCVfkzKXQ4JCoWPvdz0yTcobCFYfSBMIQilKTyqM8XKHDxFmU3WWo6x/xRGSr4ZSxeRP+WlNdUOv7VzrLTfgAXUJ24EF85DyGv98k8efE78yC0sZP1Y8cHDITJ28xKiKijENAG4ypvXeqWi7cpmK9zOJ0FzswUT6M74MZomWRJGbNWStwveg02/QZ0b4kgPLKy0QLIRmRC3hUyDgZJsFmJKUWCKNvEZwBxx4LkhBizgvsUwZJ6D+qBejfUYBiNrbyx0QbIhtY6TX8MkDa69Y+oSYbKoi7Fr2hOIrAl2+hgBzRUZxhBoh5Q70CVmLVJeluyGlJvSgkpSbp1YdGJTuytS3sQ+qzJGUq7UNZJyRbmi2JkvQGAWJdPk6ua5ceYvZzyOslnByS+TKU/z2T8S8y2ACJ3UrQEJnUIjMe8ZXOjEbg9g6DTah6ERZV7c+ti41fp9jYGHhbbW13OWLkCyd3EbrvhLwEt3mYdtFmtveVTHCjeI3XhJ8hn412wu3rKHwE0VgdEHbloIplxec+NjbpSMd6Id5UsaSWi855EXR78NNKxiTxDCux1W2QaVxrtCpe9XZBhUur0h0GFkjYQEObYVCGZOCMA+poBRV9Ann9jc9JBneo/am9h0MDTWYxhjob2wCqJ1wyqyZD9hlfZ2ebVgn1UZY1hFqWsMqyjKFcXybIY8hvKJs2yMn2hq79ZV6KRuzV3oFNoHkryt+EkjRKkTuz1UqdNoH4bGGD+pJ/kQ4id5bkseLklzx3hUx+COoZJ9DpVYE9dpYTvvvoVKGhm1Lpxac0O3fXrc3hDoMFTiYQpDy+bC8PkuwA7zAfMCKP5g3KLQtIQp7GMwNNZjGGOhxVAJqR0qIX2FSmBnoZIG9lmVMYZKlLrGUImiXFHs5+s8g3Z+LSDR1BAIJ19by2Mnp9epPGyMxcabxF+kUcyzMZCyBUeik7o1Z6JTaB/Y8rYCKb2NjG4xp06jfRga3QdS7n21MZKyi5GUueo3wWLlN/PQil/6TRCUfvOojj0e4yx7HGeB9sQ03d2OszQ0cB1Gibdj17ZPltvr0w4DJ3YYEu4jBGzumQBbPgMUcRvIDZLYtSkitJe9O431GMZ
			+ "        }";
		Map<Integer, Integer> errors = Maps.newHashMap();
		PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
		cm.setMaxTotal(nThreads);
		CloseableHttpClient client = HttpClients.custom().setConnectionManager(cm).build();
		for (int i = 40000; i < 41000; i++) {
			String orcidId = String.format("%s", i);
			String record = String.format(recordTemplate, orcidId);
			countAll++;
			if (partial == nTasks) {
				System.out
					.println(
						"Waiting for tasks to complete before resubmitting to executor (countAll = " + countAll
							+ ") . . . ");
				System.out.println("Getting replies");
				long startWait = System.currentTimeMillis();
				for (Future<Integer> res : resList) {
					if (res.get() == 200)
						countOk++;
				}
				resList.clear();
				partial = 0;
				System.out
					.println(". . . Ready to submit again after " + (System.currentTimeMillis() - startWait) + " ms");
			}
			partial++;
			Future<Integer> res = executorService.submit(() -> {
				CloseableHttpResponse responsePPOST = null;
				try {

					String url = String.format(BASE_CFG_URL, indexHost, indexName, indexType, orcidId);
					HttpPost post = new HttpPost(url);
					post.setHeader("Accept", "application/json");
					post.setHeader("Content-type", "application/json");
					StringEntity entity = new StringEntity(record);
					post.setEntity(entity);
					responsePPOST = client.execute(post);
					int statusCode = responsePPOST.getStatusLine().getStatusCode();
					switch (statusCode) {
						case 200:
						case 201:
							return statusCode;
						default:
							System.out
								.println(
									responsePPOST.getStatusLine().getStatusCode() + ": "
										+ responsePPOST.getStatusLine().getReasonPhrase());
							System.out.println("Source record causing error: " + record);
							errors.merge(statusCode, 1, Integer::sum);
							return statusCode;
					}
				} catch (ConnectException ce) {
					throw ce;
				} catch (IOException e) {
					e.printStackTrace();
					errors.merge(-1, 1, Integer::sum);
				} finally {
					if (responsePPOST != null)
						responsePPOST.close();
				}
				return -1;
			});
			resList.add(res);
		}
		executorService.shutdown();

		// now let's wait for the results. We can block ourselves here: we have nothing else to do
		System.out.println("Waiting for responses");
		for (Future<Integer> res : resList) {
			if (res.get() == 200)
				countOk++;
		}
		client.close();
		cm.shutdown();

		System.out.println("countOk: " + countOk);
		System.out.println("countAll: " + countAll);
		System.out.println("errors count: " + errors.size());
	}
}