alvinalexander.com | career | drupal | java | mac | mysql | perl | scala | uml | unix  

Lucene example source code file (MockDirectoryWrapper.java)

This example Lucene source code file (MockDirectoryWrapper.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" TM.

Java - Lucene tags/keywords

hashset, hashset, indexoutput, io, ioexception, ioexception, mockdirectorywrapper, override, override, ramdirectory, ramfile, runtimeexception, set, string, string, util

The Lucene MockDirectoryWrapper.java source code

package org.apache.lucene.store;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThrottledIndexOutput;
import org.apache.lucene.util._TestUtil;

/**
 * This is a Directory Wrapper that adds methods
 * intended to be used only by unit tests.
 * It also adds a number of features useful for testing:
 * <ul>
 *   <li> Instances created by {@link LuceneTestCase#newDirectory()} are tracked 
 *        to ensure they are closed by the test.
 *   <li> When a MockDirectoryWrapper is closed, it will throw an exception if 
 *        it has any open files against it (with a stacktrace indicating where 
 *        they were opened from).
 *   <li> When a MockDirectoryWrapper is closed, it runs CheckIndex to test if
 *        the index was corrupted.
 *   <li> MockDirectoryWrapper simulates some "features" of Windows, such as
 *        refusing to write/delete to open files.
 * </ul>
 */

public class MockDirectoryWrapper extends Directory {
  final Directory delegate;
  long maxSize;

  // Max actual bytes used. This is set by MockRAMOutputStream:
  long maxUsedSize;
  double randomIOExceptionRate;
  Random randomState;
  boolean noDeleteOpenFile = true;
  boolean preventDoubleWrite = true;
  boolean checkIndexOnClose = true;
  boolean trackDiskUsage = false;
  private Set<String> unSyncedFiles;
  private Set<String> createdFiles;
  private Set<String> openFilesForWrite = new HashSet();
  Set<String> openLocks = Collections.synchronizedSet(new HashSet());
  volatile boolean crashed;
  private ThrottledIndexOutput throttledOutput;
  private Throttling throttling = Throttling.SOMETIMES;

  // use this for tracking files for crash.
  // additionally: provides debugging information in case you leave one open
  private Map<Closeable,Exception> openFileHandles = Collections.synchronizedMap(new IdentityHashMap());

  // NOTE: we cannot initialize the Map here due to the
  // order in which our constructor actually does this
  // member initialization vs when it calls super.  It seems
  // like super is called, then our members are initialized:
  private Map<String,Integer> openFiles;

  // Only tracked if noDeleteOpenFile is true: if an attempt
  // is made to delete an open file, we enroll it here.
  private Set<String> openFilesDeleted;

  private synchronized void init() {
    if (openFiles == null) {
      openFiles = new HashMap<String,Integer>();
      openFilesDeleted = new HashSet<String>();
    }

    if (createdFiles == null)
      createdFiles = new HashSet<String>();
    if (unSyncedFiles == null)
      unSyncedFiles = new HashSet<String>();
  }

  public MockDirectoryWrapper(Random random, Directory delegate) {
    this.delegate = delegate;
    // must make a private random since our methods are
    // called from different threads; else test failures may
    // not be reproducible from the original seed
    this.randomState = new Random(random.nextInt());
    this.throttledOutput = new ThrottledIndexOutput(ThrottledIndexOutput
        .mBitsToBytes(40 + randomState.nextInt(10)), 5 + randomState.nextInt(5), null);
    // force wrapping of lockfactory
    try {
      setLockFactory(new MockLockFactoryWrapper(this, delegate.getLockFactory()));
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    init();
  }

  public void setTrackDiskUsage(boolean v) {
    trackDiskUsage = v;
  }

  /** If set to true, we throw an IOException if the same
   *  file is opened by createOutput, ever. */
  public void setPreventDoubleWrite(boolean value) {
    preventDoubleWrite = value;
  }

  @Deprecated
  @Override
  public void sync(String name) throws IOException {
    maybeYield();
    maybeThrowDeterministicException();
    if (crashed)
      throw new IOException("cannot sync after crash");
    unSyncedFiles.remove(name);
    delegate.sync(name);
  }
  
  public static enum Throttling {
    /** always emulate a slow hard disk. could be very slow! */
    ALWAYS,
    /** sometimes (2% of the time) emulate a slow hard disk. */
    SOMETIMES,
    /** never throttle output */
    NEVER
  }
  
  public void setThrottling(Throttling throttling) {
    this.throttling = throttling;
  }

  @Override
  public synchronized void sync(Collection<String> names) throws IOException {
    maybeYield();
    for (String name : names)
      maybeThrowDeterministicException();
    if (crashed)
      throw new IOException("cannot sync after crash");
    unSyncedFiles.removeAll(names);
    delegate.sync(names);
  }
  
  @Override
  public String toString() {
    maybeYield();
    return "MockDirWrapper(" + delegate + ")";
  }

  public synchronized final long sizeInBytes() throws IOException {
    if (delegate instanceof RAMDirectory)
      return ((RAMDirectory) delegate).sizeInBytes();
    else {
      // hack
      long size = 0;
      for (String file : delegate.listAll())
        size += delegate.fileLength(file);
      return size;
    }
  }

  /** Simulates a crash of OS or machine by overwriting
   *  unsynced files. */
  public synchronized void crash() throws IOException {
    crashed = true;
    openFiles = new HashMap<String,Integer>();
    openFilesForWrite = new HashSet<String>();
    openFilesDeleted = new HashSet<String>();
    Iterator<String> it = unSyncedFiles.iterator();
    unSyncedFiles = new HashSet<String>();
    // first force-close all files, so we can corrupt on windows etc.
    // clone the file map, as these guys want to remove themselves on close.
    Map<Closeable,Exception> m = new IdentityHashMap(openFileHandles);
    for (Closeable f : m.keySet())
      try {
        f.close();
      } catch (Exception ignored) {}
    
    int count = 0;
    while(it.hasNext()) {
      String name = it.next();
      if (count % 3 == 0) {
        deleteFile(name, true);
      } else if (count % 3 == 1) {
        // Zero out file entirely
        long length = fileLength(name);
        byte[] zeroes = new byte[256];
        long upto = 0;
        IndexOutput out = delegate.createOutput(name);
        while(upto < length) {
          final int limit = (int) Math.min(length-upto, zeroes.length);
          out.writeBytes(zeroes, 0, limit);
          upto += limit;
        }
        out.close();
      } else if (count % 3 == 2) {
        // Truncate the file:
        IndexOutput out = delegate.createOutput(name);
        out.setLength(fileLength(name)/2);
        out.close();
      }
      count++;
    }
  }

  public synchronized void clearCrash() throws IOException {
    crashed = false;
    openLocks.clear();
  }

  public void setMaxSizeInBytes(long maxSize) {
    this.maxSize = maxSize;
  }
  public long getMaxSizeInBytes() {
    return this.maxSize;
  }

  /**
   * Returns the peek actual storage used (bytes) in this
   * directory.
   */
  public long getMaxUsedSizeInBytes() {
    return this.maxUsedSize;
  }
  public void resetMaxUsedSizeInBytes() throws IOException {
    this.maxUsedSize = getRecomputedActualSizeInBytes();
  }

  /**
   * Emulate windows whereby deleting an open file is not
   * allowed (raise IOException).
  */
  public void setNoDeleteOpenFile(boolean value) {
    this.noDeleteOpenFile = value;
  }
  public boolean getNoDeleteOpenFile() {
    return noDeleteOpenFile;
  }

  /**
   * Set whether or not checkindex should be run
   * on close
   */
  public void setCheckIndexOnClose(boolean value) {
    this.checkIndexOnClose = value;
  }
  
  public boolean getCheckIndexOnClose() {
    return checkIndexOnClose;
  }
  /**
   * If 0.0, no exceptions will be thrown.  Else this should
   * be a double 0.0 - 1.0.  We will randomly throw an
   * IOException on the first write to an OutputStream based
   * on this probability.
   */
  public void setRandomIOExceptionRate(double rate) {
    randomIOExceptionRate = rate;
  }
  public double getRandomIOExceptionRate() {
    return randomIOExceptionRate;
  }

  void maybeThrowIOException() throws IOException {
    if (randomIOExceptionRate > 0.0) {
      int number = Math.abs(randomState.nextInt() % 1000);
      if (number < randomIOExceptionRate*1000) {
        if (LuceneTestCase.VERBOSE) {
          System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception");
          new Throwable().printStackTrace(System.out);
        }
        throw new IOException("a random IOException");
      }
    }
  }

  @Override
  public synchronized void deleteFile(String name) throws IOException {
    maybeYield();
    deleteFile(name, false);
  }

  // sets the cause of the incoming ioe to be the stack
  // trace when the offending file name was opened
  private synchronized IOException fillOpenTrace(IOException ioe, String name, boolean input) {
    for(Map.Entry<Closeable,Exception> ent : openFileHandles.entrySet()) {
      if (input && ent.getKey() instanceof MockIndexInputWrapper && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) {
        ioe.initCause(ent.getValue());
        break;
      } else if (!input && ent.getKey() instanceof MockIndexOutputWrapper && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) {
        ioe.initCause(ent.getValue());
        break;
      }
    }
    return ioe;
  }

  private void maybeYield() {
    if (randomState.nextBoolean()) {
      Thread.yield();
    }
  }

  private synchronized void deleteFile(String name, boolean forced) throws IOException {
    maybeYield();

    maybeThrowDeterministicException();

    if (crashed && !forced)
      throw new IOException("cannot delete after crash");

    if (unSyncedFiles.contains(name))
      unSyncedFiles.remove(name);
    if (!forced && noDeleteOpenFile) {
      if (openFiles.containsKey(name)) {
        openFilesDeleted.add(name);
        throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
      } else {
        openFilesDeleted.remove(name);
      }
    }
    delegate.deleteFile(name);
  }

  public synchronized Set<String> getOpenDeletedFiles() {
    return new HashSet<String>(openFilesDeleted);
  }

  @Override
  public synchronized IndexOutput createOutput(String name) throws IOException {
    maybeYield();
    if (crashed)
      throw new IOException("cannot createOutput after crash");
    init();
    synchronized(this) {
      if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen"))
        throw new IOException("file \"" + name + "\" was already written to");
    }
    if (noDeleteOpenFile && openFiles.containsKey(name))
      throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
    
    if (crashed)
      throw new IOException("cannot createOutput after crash");
    unSyncedFiles.add(name);
    createdFiles.add(name);
    
    if (delegate instanceof RAMDirectory) {
      RAMDirectory ramdir = (RAMDirectory) delegate;
      RAMFile file = new RAMFile(ramdir);
      RAMFile existing = ramdir.fileMap.get(name);
    
      // Enforce write once:
      if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite)
        throw new IOException("file " + name + " already exists");
      else {
        if (existing!=null) {
          ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes);
          existing.directory = null;
        }
        ramdir.fileMap.put(name, file);
      }
    }
    
    //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
    IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name);
    addFileHandle(io, name, false);
    openFilesForWrite.add(name);
    
    // throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
    if (throttling == Throttling.ALWAYS || 
        (throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0)) {
      if (LuceneTestCase.VERBOSE) {
        System.out.println("MockDirectoryWrapper: throttling indexOutput");
      }
      return throttledOutput.newFromDelegate(io);
    } else {
      return io;
    }
  }

  private void addFileHandle(Closeable c, String name, boolean input) {
    Integer v = openFiles.get(name);
    if (v != null) {
      v = Integer.valueOf(v.intValue()+1);
      openFiles.put(name, v);
    } else {
      openFiles.put(name, Integer.valueOf(1));
    }
    
    openFileHandles.put(c, new RuntimeException("unclosed Index" + (input ? "Input" : "Output") + ": " + name));
  }
  
  @Override
  public synchronized IndexInput openInput(String name) throws IOException {
    maybeYield();
    if (!delegate.fileExists(name))
      throw new FileNotFoundException(name);

    // cannot open a file for input if it's still open for
    // output, except for segments.gen and segments_N
    if (openFilesForWrite.contains(name) && !name.startsWith("segments")) {
      throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
    }

    IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name));
    addFileHandle(ii, name, true);
    return ii;
  }

  /** Provided for testing purposes.  Use sizeInBytes() instead. */
  public synchronized final long getRecomputedSizeInBytes() throws IOException {
    if (!(delegate instanceof RAMDirectory))
      return sizeInBytes();
    long size = 0;
    for(final RAMFile file: ((RAMDirectory)delegate).fileMap.values()) {
      size += file.getSizeInBytes();
    }
    return size;
  }

  /** Like getRecomputedSizeInBytes(), but, uses actual file
   * lengths rather than buffer allocations (which are
   * quantized up to nearest
   * RAMOutputStream.BUFFER_SIZE (now 1024) bytes.
   */

  public final synchronized long getRecomputedActualSizeInBytes() throws IOException {
    if (!(delegate instanceof RAMDirectory))
      return sizeInBytes();
    long size = 0;
    for (final RAMFile file : ((RAMDirectory)delegate).fileMap.values())
      size += file.length;
    return size;
  }

  @Override
  public synchronized void close() throws IOException {
    maybeYield();
    if (openFiles == null) {
      openFiles = new HashMap<String,Integer>();
      openFilesDeleted = new HashSet<String>();
    }
    if (noDeleteOpenFile && openFiles.size() > 0) {
      // print the first one as its very verbose otherwise
      Exception cause = null;
      Iterator<Exception> stacktraces = openFileHandles.values().iterator();
      if (stacktraces.hasNext())
        cause = stacktraces.next();
      // RuntimeException instead of IOException because
      // super() does not throw IOException currently:
      throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause);
    }
    if (noDeleteOpenFile && openLocks.size() > 0) {
      throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks);
    }
    open = false;
    if (checkIndexOnClose && IndexReader.indexExists(this)) {
      if (LuceneTestCase.VERBOSE) {
        System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
      } 
      _TestUtil.checkIndex(this);
    }
    delegate.close();
  }

  private synchronized void removeOpenFile(Closeable c, String name) {
    Integer v = openFiles.get(name);
    // Could be null when crash() was called
    if (v != null) {
      if (v.intValue() == 1) {
        openFiles.remove(name);
        openFilesDeleted.remove(name);
      } else {
        v = Integer.valueOf(v.intValue()-1);
        openFiles.put(name, v);
      }
    }

    openFileHandles.remove(c);
  }
  
  public synchronized void removeIndexOutput(IndexOutput out, String name) {
    openFilesForWrite.remove(name);
    removeOpenFile(out, name);
  }
  
  public synchronized void removeIndexInput(IndexInput in, String name) {
    removeOpenFile(in, name);
  }
  
  boolean open = true;
  
  public synchronized boolean isOpen() {
    return open;
  }
  
  /**
   * Objects that represent fail-able conditions. Objects of a derived
   * class are created and registered with the mock directory. After
   * register, each object will be invoked once for each first write
   * of a file, giving the object a chance to throw an IOException.
   */
  public static class Failure {
    /**
     * eval is called on the first write of every new file.
     */
    public void eval(MockDirectoryWrapper dir) throws IOException { }

    /**
     * reset should set the state of the failure to its default
     * (freshly constructed) state. Reset is convenient for tests
     * that want to create one failure object and then reuse it in
     * multiple cases. This, combined with the fact that Failure
     * subclasses are often anonymous classes makes reset difficult to
     * do otherwise.
     *
     * A typical example of use is
     * Failure failure = new Failure() { ... };
     * ...
     * mock.failOn(failure.reset())
     */
    public Failure reset() { return this; }

    protected boolean doFail;

    public void setDoFail() {
      doFail = true;
    }

    public void clearDoFail() {
      doFail = false;
    }
  }

  ArrayList<Failure> failures;

  /**
   * add a Failure object to the list of objects to be evaluated
   * at every potential failure point
   */
  synchronized public void failOn(Failure fail) {
    if (failures == null) {
      failures = new ArrayList<Failure>();
    }
    failures.add(fail);
  }

  /**
   * Iterate through the failures list, giving each object a
   * chance to throw an IOE
   */
  synchronized void maybeThrowDeterministicException() throws IOException {
    if (failures != null) {
      for(int i = 0; i < failures.size(); i++) {
        failures.get(i).eval(this);
      }
    }
  }

  @Override
  public synchronized String[] listAll() throws IOException {
    maybeYield();
    return delegate.listAll();
  }

  @Override
  public synchronized boolean fileExists(String name) throws IOException {
    maybeYield();
    return delegate.fileExists(name);
  }

  @Override
  public synchronized long fileModified(String name) throws IOException {
    maybeYield();
    return delegate.fileModified(name);
  }

  @Override
  @Deprecated
  /*  @deprecated Lucene never uses this API; it will be
   *  removed in 4.0. */
  public synchronized void touchFile(String name) throws IOException {
    maybeYield();
    delegate.touchFile(name);
  }

  @Override
  public synchronized long fileLength(String name) throws IOException {
    maybeYield();
    return delegate.fileLength(name);
  }

  @Override
  public synchronized Lock makeLock(String name) {
    maybeYield();
    return delegate.makeLock(name);
  }

  @Override
  public synchronized void clearLock(String name) throws IOException {
    maybeYield();
    delegate.clearLock(name);
  }

  @Override
  public synchronized void setLockFactory(LockFactory lockFactory) throws IOException {
    maybeYield();
    delegate.setLockFactory(lockFactory);
  }

  @Override
  public synchronized LockFactory getLockFactory() {
    maybeYield();
    return delegate.getLockFactory();
  }

  @Override
  public synchronized String getLockID() {
    maybeYield();
    return delegate.getLockID();
  }

  @Override
  public synchronized void copy(Directory to, String src, String dest) throws IOException {
    maybeYield();
    delegate.copy(to, src, dest);
  }
}

Other Lucene examples (source code examples)

Here is a short list of links related to this Lucene MockDirectoryWrapper.java source code file:

... this post is sponsored by my books ...

#1 New Release!

FP Best Seller

 

new blog posts

 

Copyright 1998-2021 Alvin Alexander, alvinalexander.com
All Rights Reserved.

A percentage of advertising revenue from
pages under the /java/jwarehouse URI on this website is
paid back to open source projects.