package eu.dnetlib.data.mdstore.modular.mongodb; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.Iterator; import java.util.UUID; import com.mongodb.DBObject; import com.mongodb.client.MongoDatabase; import eu.dnetlib.data.mdstore.MDStoreServiceException; import eu.dnetlib.data.mdstore.modular.RecordParserFactory; import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao; import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @Ignore @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = ConfigurationTestConfig.class) public class FeedSpeedTest { private static final int N_RECORDS = 68271; @Autowired private MongoDatabase db; @Autowired private MDStoreDao dao; @Autowired private RecordParserFactory recordParserfactory; @Before public void setup() throws MDStoreServiceException { dao.createMDStore("speed_test", "testFormat", "testInterpretation", "testLayout"); } @Test public void testSpeedFromFolder() throws IOException { Iterable iterable = new Iterable() { private int counter = 0; private double last = System.currentTimeMillis(); @Override public Iterator iterator() { return new Iterator() { @Override public boolean hasNext() { return counter < N_RECORDS; } @Override public String next() { if (counter % 10000 == 0) { System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds"); last = System.currentTimeMillis(); } File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++)); if (f.exists()) { try { FileInputStream fileInputStream = new FileInputStream(f); String s = IOUtils.toString(fileInputStream); fileInputStream.close(); return s; } catch (Exception e) { return null; } } else { try { FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000001.xml")); String s = IOUtils.toString(fileInputStream); fileInputStream.close(); return s; } catch (Exception e) { return null; } } } @Override public void remove() {} }; } }; MongoMDStore mdStore = new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), true, db); mdStore.feed(iterable, false); } //@Ignore @Test public void testFeedSpeedFromTemplate() throws MDStoreServiceException, IOException { MongoMDStore mdStore = new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), false, db); mdStore.feed(new Iterable() { private int counter = 0; private double last = System.currentTimeMillis(); private String templateRecord = IOUtils.toString(new ClassPathResource("/eu/dnetlib/data/mdstore/modular/mongodb/templateRecord.xml") .getInputStream()); @Override public Iterator iterator() { return new Iterator() { @Override public boolean hasNext() { return counter < N_RECORDS; } @Override public String next() { if (counter % 10000 == 0) { System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds"); last = System.currentTimeMillis(); } File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++)); if (f.exists()) { try { FileInputStream fileInputStream = new FileInputStream(f); String s = IOUtils.toString(fileInputStream); fileInputStream.close(); return s; } catch (Exception e) { return null; } } else { counter++; try { FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000009.xml")); String s = IOUtils.toString(fileInputStream); fileInputStream.close(); return s; } catch (Exception e) { return null; } } } @Override public void remove() {} }; } }, false); } }