alvinalexander.com | career | drupal | java | mac | mysql | perl | scala | uml | unix  

Lucene example source code file (TestBufferedIndexInput.java)

This example Lucene source code file (TestBufferedIndexInput.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" TM.

Java - Lucene tags/keywords

bufferedindexinput, indexinput, io, ioexception, ioexception, mockfsdirectory, mybufferedindexinput, mybufferedindexinput, override, override, random, term, term, termquery, test_file_length, util

The Lucene TestBufferedIndexInput.java source code

package org.apache.lucene.store;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;

import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.NIOFSDirectory.NIOFSIndexInput;
import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.ArrayUtil;

public class TestBufferedIndexInput extends LuceneTestCase {
  
  private static void writeBytes(File aFile, long size) throws IOException{
    OutputStream stream = null;
    try {
      stream = new FileOutputStream(aFile);
      for (int i = 0; i < size; i++) {
        stream.write(byten(i));  
      }
      stream.flush();
    } finally {
      if (stream != null) {
        stream.close();
      }
    }
  }

  private static final long TEST_FILE_LENGTH = 100*1024;
 
  // Call readByte() repeatedly, past the buffer boundary, and see that it
  // is working as expected.
  // Our input comes from a dynamically generated/ "file" - see
  // MyBufferedIndexInput below.
  public void testReadByte() throws Exception {
    MyBufferedIndexInput input = new MyBufferedIndexInput();
    for (int i = 0; i < BufferedIndexInput.BUFFER_SIZE * 10; i++) {
      assertEquals(input.readByte(), byten(i));
    }
  }
 
  // Call readBytes() repeatedly, with various chunk sizes (from 1 byte to
  // larger than the buffer size), and see that it returns the bytes we expect.
  // Our input comes from a dynamically generated "file" -
  // see MyBufferedIndexInput below.
  public void testReadBytes() throws Exception {
    MyBufferedIndexInput input = new MyBufferedIndexInput();
    runReadBytes(input, BufferedIndexInput.BUFFER_SIZE, random);

    // This tests the workaround code for LUCENE-1566 where readBytesInternal
    // provides a workaround for a JVM Bug that incorrectly raises a OOM Error
    // when a large byte buffer is passed to a file read.
    // NOTE: this does only test the chunked reads and NOT if the Bug is triggered.
    //final int tmpFileSize = 1024 * 1024 * 5;
    final int inputBufferSize = 128;
    File tmpInputFile = File.createTempFile("IndexInput", "tmpFile");
    tmpInputFile.deleteOnExit();
    writeBytes(tmpInputFile, TEST_FILE_LENGTH);

    // run test with chunk size of 10 bytes
    runReadBytesAndClose(new SimpleFSIndexInput(tmpInputFile,
                                                inputBufferSize, 10), inputBufferSize, random);

    // run test with chunk size of 10 bytes
    runReadBytesAndClose(new NIOFSIndexInput(tmpInputFile,
                                             inputBufferSize, 10), inputBufferSize, random);
  }

  private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r)
      throws IOException {
    try {
      runReadBytes(input, bufferSize, r);
    } finally {
      input.close();
    }
  }
  
  private void runReadBytes(IndexInput input, int bufferSize, Random r)
      throws IOException {

    int pos = 0;
    // gradually increasing size:
    for (int size = 1; size < bufferSize * 10; size = size + size / 200 + 1) {
      checkReadBytes(input, size, pos);
      pos += size;
      if (pos >= TEST_FILE_LENGTH) {
        // wrap
        pos = 0;
        input.seek(0L);
      }
    }
    // wildly fluctuating size:
    for (long i = 0; i < 100; i++) {
      final int size = r.nextInt(10000);
      checkReadBytes(input, 1+size, pos);
      pos += 1+size;
      if (pos >= TEST_FILE_LENGTH) {
        // wrap
        pos = 0;
        input.seek(0L);
      }
    }
    // constant small size (7 bytes):
    for (int i = 0; i < bufferSize; i++) {
      checkReadBytes(input, 7, pos);
      pos += 7;
      if (pos >= TEST_FILE_LENGTH) {
        // wrap
        pos = 0;
        input.seek(0L);
      }
    }
  }

  private byte[] buffer = new byte[10];
    
  private void checkReadBytes(IndexInput input, int size, int pos) throws IOException{
    // Just to see that "offset" is treated properly in readBytes(), we
    // add an arbitrary offset at the beginning of the array
    int offset = size % 10; // arbitrary
    buffer = ArrayUtil.grow(buffer, offset+size);
    assertEquals(pos, input.getFilePointer());
    long left = TEST_FILE_LENGTH - input.getFilePointer();
    if (left <= 0) {
      return;
    } else if (left < size) {
      size = (int) left;
    }
    input.readBytes(buffer, offset, size);
    assertEquals(pos+size, input.getFilePointer());
    for(int i=0; i<size; i++) {
      assertEquals("pos=" + i + " filepos=" + (pos+i), byten(pos+i), buffer[offset+i]);
    }
  }
   
  // This tests that attempts to readBytes() past an EOF will fail, while
  // reads up to the EOF will succeed. The EOF is determined by the
  // BufferedIndexInput's arbitrary length() value.
  public void testEOF() throws Exception {
     MyBufferedIndexInput input = new MyBufferedIndexInput(1024);
     // see that we can read all the bytes at one go:
     checkReadBytes(input, (int)input.length(), 0);  
     // go back and see that we can't read more than that, for small and
     // large overflows:
     int pos = (int)input.length()-10;
     input.seek(pos);
     checkReadBytes(input, 10, pos);  
     input.seek(pos);
     try {
       checkReadBytes(input, 11, pos);
           fail("Block read past end of file");
       } catch (IOException e) {
           /* success */
       }
     input.seek(pos);
     try {
       checkReadBytes(input, 50, pos);
           fail("Block read past end of file");
       } catch (IOException e) {
           /* success */
       }
     input.seek(pos);
     try {
       checkReadBytes(input, 100000, pos);
           fail("Block read past end of file");
       } catch (IOException e) {
           /* success */
       }
  }

    // byten emulates a file - byten(n) returns the n'th byte in that file.
    // MyBufferedIndexInput reads this "file".
    private static byte byten(long n){
      return (byte)(n*n%256);
    }
    private static class MyBufferedIndexInput extends BufferedIndexInput {
      private long pos;
      private long len;
      public MyBufferedIndexInput(long len){
        this.len = len;
        this.pos = 0;
      }
      public MyBufferedIndexInput(){
        // an infinite file
        this(Long.MAX_VALUE);
      }
      @Override
      protected void readInternal(byte[] b, int offset, int length) throws IOException {
        for(int i=offset; i<offset+length; i++)
          b[i] = byten(pos++);
      }

      @Override
      protected void seekInternal(long pos) throws IOException {
        this.pos = pos;
      }

      @Override
      public void close() throws IOException {
      }

      @Override
      public long length() {
        return len;
      }
    }

    public void testSetBufferSize() throws IOException {
      File indexDir = _TestUtil.getTempDir("testSetBufferSize");
      MockFSDirectory dir = new MockFSDirectory(indexDir, random);
      try {
        IndexWriter writer = new IndexWriter(
            dir,
            new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
                setOpenMode(OpenMode.CREATE).
                setMergePolicy(newLogMergePolicy(false))
        );
        for(int i=0;i<37;i++) {
          Document doc = new Document();
          doc.add(newField("content", "aaa bbb ccc ddd" + i, Field.Store.YES, Field.Index.ANALYZED));
          doc.add(newField("id", "" + i, Field.Store.YES, Field.Index.ANALYZED));
          writer.addDocument(doc);
        }
        writer.close();

        dir.allIndexInputs.clear();

        IndexReader reader = IndexReader.open(dir, false);
        Term aaa = new Term("content", "aaa");
        Term bbb = new Term("content", "bbb");
        Term ccc = new Term("content", "ccc");
        assertEquals(37, reader.docFreq(ccc));
        reader.deleteDocument(0);
        assertEquals(37, reader.docFreq(aaa));
        dir.tweakBufferSizes();
        reader.deleteDocument(4);
        assertEquals(reader.docFreq(bbb), 37);
        dir.tweakBufferSizes();

        IndexSearcher searcher = newSearcher(reader);
        ScoreDoc[] hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs;
        dir.tweakBufferSizes();
        assertEquals(35, hits.length);
        dir.tweakBufferSizes();
        hits = searcher.search(new TermQuery(new Term("id", "33")), null, 1000).scoreDocs;
        dir.tweakBufferSizes();
        assertEquals(1, hits.length);
        hits = searcher.search(new TermQuery(aaa), null, 1000).scoreDocs;
        dir.tweakBufferSizes();
        assertEquals(35, hits.length);
        searcher.close();
        reader.close();
      } finally {
        _TestUtil.rmDir(indexDir);
      }
    }

    private static class MockFSDirectory extends Directory {

      List<IndexInput> allIndexInputs = new ArrayList();

      Random rand;

      private Directory dir;

      public MockFSDirectory(File path, Random rand) throws IOException {
        this.rand = rand;
        lockFactory = NoLockFactory.getNoLockFactory();
        dir = new SimpleFSDirectory(path, null);
      }

      @Override
      public IndexInput openInput(String name) throws IOException {
        return openInput(name, BufferedIndexInput.BUFFER_SIZE);
      }

      public void tweakBufferSizes() {
        //int count = 0;
        for (final IndexInput ip : allIndexInputs) {
          BufferedIndexInput bii = (BufferedIndexInput) ip;
          int bufferSize = 1024+Math.abs(rand.nextInt() % 32768);
          bii.setBufferSize(bufferSize);
          //count++;
        }
        //System.out.println("tweak'd " + count + " buffer sizes");
      }
      
      @Override
      public IndexInput openInput(String name, int bufferSize) throws IOException {
        // Make random changes to buffer size
        bufferSize = 1+Math.abs(rand.nextInt() % 10);
        IndexInput f = dir.openInput(name, bufferSize);
        allIndexInputs.add(f);
        return f;
      }

      @Override
      public IndexOutput createOutput(String name) throws IOException {
        return dir.createOutput(name);
      }

      @Override
      public void close() throws IOException {
        dir.close();
      }

      @Override
      public void deleteFile(String name)
        throws IOException
      {
        dir.deleteFile(name);
      }
      @Override
      @Deprecated
      /*  @deprecated Lucene never uses this API; it will be
       *  removed in 4.0. */
      public void touchFile(String name)
        throws IOException
      {
        dir.touchFile(name);
      }
      @Override
      public long fileModified(String name)
        throws IOException
      {
        return dir.fileModified(name);
      }
      @Override
      public boolean fileExists(String name)
        throws IOException
      {
        return dir.fileExists(name);
      }
      @Override
      public String[] listAll()
        throws IOException
      {
        return dir.listAll();
      }

      @Override
      public long fileLength(String name) throws IOException {
        return dir.fileLength(name);
      }


    }
}

Other Lucene examples (source code examples)

Here is a short list of links related to this Lucene TestBufferedIndexInput.java source code file:

... this post is sponsored by my books ...

#1 New Release!

FP Best Seller

 

new blog posts

 

Copyright 1998-2021 Alvin Alexander, alvinalexander.com
All Rights Reserved.

A percentage of advertising revenue from
pages under the /java/jwarehouse URI on this website is
paid back to open source projects.