dnet-core/dnet-data-services/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/FeedSpeedTest.java

package eu.dnetlib.data.mdstore.modular.mongodb;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Iterator;
import java.util.UUID;

import com.mongodb.DBObject;
import com.mongodb.client.MongoDatabase;
import eu.dnetlib.data.mdstore.MDStoreServiceException;
import eu.dnetlib.data.mdstore.modular.RecordParserFactory;
import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@Ignore
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = ConfigurationTestConfig.class)
public class FeedSpeedTest {

	private static final int N_RECORDS = 68271;

	@Autowired
	private MongoDatabase db;

	@Autowired
	private MDStoreDao dao;

	@Autowired
	private RecordParserFactory recordParserfactory;

	@Before
	public void setup() throws MDStoreServiceException {
		dao.createMDStore("speed_test", "testFormat", "testInterpretation", "testLayout");
	}


	@Test
	public void testSpeedFromFolder() throws IOException {
		Iterable<String> iterable = new Iterable<String>() {

			private int counter = 0;
			private double last = System.currentTimeMillis();

			@Override
			public Iterator<String> iterator() {
				return new Iterator<String>() {

					@Override
					public boolean hasNext() {
						return counter < N_RECORDS;
					}

					@Override
					public String next() {
						if (counter % 10000 == 0) {
							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");
							last = System.currentTimeMillis();
						}

						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));
						if (f.exists()) {
							try {
								FileInputStream fileInputStream = new FileInputStream(f);
								String s = IOUtils.toString(fileInputStream);
								fileInputStream.close();
								return s;
							} catch (Exception e) {
								return null;
							}
						} else {
							try {
								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000001.xml"));
								String s = IOUtils.toString(fileInputStream);
								fileInputStream.close();
								return s;
							} catch (Exception e) {
								return null;
							}
						}
					}

					@Override
					public void remove() {}
				};
			}
		};

		MongoMDStore mdStore =
				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), true, db);
		mdStore.feed(iterable, false);
	}

	//@Ignore
	@Test
	public void testFeedSpeedFromTemplate() throws MDStoreServiceException, IOException {
		MongoMDStore mdStore =
				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), false, db);
		mdStore.feed(new Iterable<String>() {

			private int counter = 0;
			private double last = System.currentTimeMillis();
			private String templateRecord = IOUtils.toString(new ClassPathResource("/eu/dnetlib/data/mdstore/modular/mongodb/templateRecord.xml")
					.getInputStream());

			@Override
			public Iterator<String> iterator() {
				return new Iterator<String>() {

					@Override
					public boolean hasNext() {
						return counter < N_RECORDS;
					}

					@Override
					public String next() {
						if (counter % 10000 == 0) {
							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");
							last = System.currentTimeMillis();
						}

						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));
						if (f.exists()) {
							try {
								FileInputStream fileInputStream = new FileInputStream(f);
								String s = IOUtils.toString(fileInputStream);
								fileInputStream.close();
								return s;
							} catch (Exception e) {
								return null;
							}
						} else {
							counter++;
							try {
								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000009.xml"));
								String s = IOUtils.toString(fileInputStream);
								fileInputStream.close();
								return s;
							} catch (Exception e) {
								return null;
							}
						}
					}

					@Override
					public void remove() {}
				};
			}
		}, false);
	}
}