home | career | drupal | java | mac | mysql | perl | scala | uml | unix  

Lucene example source code file (TestDeletionPolicy.java)

This example Lucene source code file (TestDeletionPolicy.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" TM.

Java - Lucene tags/keywords

directory, indexcommit, indexcommit, indexreader, indexwriter, indexwriter, io, ioexception, ioexception, keeplastndeletionpolicy, logmergepolicy, mockanalyzer, mockanalyzer, n, test_version_current, util

The Lucene TestDeletionPolicy.java source code

package org.apache.lucene.index;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.Collection;

import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;

/*
  Verify we can read the pre-2.1 file format, do searches
  against it, and add documents to it.
*/

public class TestDeletionPolicy extends LuceneTestCase {
  
  private void verifyCommitOrder(List<? extends IndexCommit> commits) throws IOException {
    final IndexCommit firstCommit =  commits.get(0);
    long last = SegmentInfos.generationFromSegmentsFileName(firstCommit.getSegmentsFileName());
    assertEquals(last, firstCommit.getGeneration());
    long lastVersion = firstCommit.getVersion();
    long lastTimestamp = firstCommit.getTimestamp();
    for(int i=1;i<commits.size();i++) {
      final IndexCommit commit =  commits.get(i);
      long now = SegmentInfos.generationFromSegmentsFileName(commit.getSegmentsFileName());
      long nowVersion = commit.getVersion();
      long nowTimestamp = commit.getTimestamp();
      assertTrue("SegmentInfos commits are out-of-order", now > last);
      assertTrue("SegmentInfos versions are out-of-order", nowVersion > lastVersion);
      assertTrue("SegmentInfos timestamps are out-of-order: now=" + nowTimestamp + " vs last=" + lastTimestamp, nowTimestamp >= lastTimestamp);
      assertEquals(now, commit.getGeneration());
      last = now;
      lastVersion = nowVersion;
      lastTimestamp = nowTimestamp;
    }
  }

  class KeepAllDeletionPolicy implements IndexDeletionPolicy {
    int numOnInit;
    int numOnCommit;
    Directory dir;
    public void onInit(List<? extends IndexCommit> commits) throws IOException {
      verifyCommitOrder(commits);
      numOnInit++;
    }
    public void onCommit(List<? extends IndexCommit> commits) throws IOException {
      IndexCommit lastCommit =  commits.get(commits.size()-1);
      IndexReader r = IndexReader.open(dir, true);
      assertEquals("lastCommit.isOptimized()=" + lastCommit.isOptimized() + " vs IndexReader.isOptimized=" + r.isOptimized(), r.isOptimized(), lastCommit.isOptimized());
      r.close();
      verifyCommitOrder(commits);
      numOnCommit++;
    }
  }

  /**
   * This is useful for adding to a big index when you know
   * readers are not using it.
   */
  class KeepNoneOnInitDeletionPolicy implements IndexDeletionPolicy {
    int numOnInit;
    int numOnCommit;
    public void onInit(List<? extends IndexCommit> commits) throws IOException {
      verifyCommitOrder(commits);
      numOnInit++;
      // On init, delete all commit points:
      for (final IndexCommit commit : commits) {
        commit.delete();
        assertTrue(commit.isDeleted());
      }
    }
    public void onCommit(List<? extends IndexCommit> commits) throws IOException {
      verifyCommitOrder(commits);
      int size = commits.size();
      // Delete all but last one:
      for(int i=0;i<size-1;i++) {
        ((IndexCommit) commits.get(i)).delete();
      }
      numOnCommit++;
    }
  }

  class KeepLastNDeletionPolicy implements IndexDeletionPolicy {
    int numOnInit;
    int numOnCommit;
    int numToKeep;
    int numDelete;
    Set<String> seen = new HashSet();

    public KeepLastNDeletionPolicy(int numToKeep) {
      this.numToKeep = numToKeep;
    }

    public void onInit(List<? extends IndexCommit> commits) throws IOException {
      if (VERBOSE) {
        System.out.println("TEST: onInit");
      }
      verifyCommitOrder(commits);
      numOnInit++;
      // do no deletions on init
      doDeletes(commits, false);
    }

    public void onCommit(List<? extends IndexCommit> commits) throws IOException {
      if (VERBOSE) {
        System.out.println("TEST: onCommit");
      }
      verifyCommitOrder(commits);
      doDeletes(commits, true);
    }
    
    private void doDeletes(List<? extends IndexCommit> commits, boolean isCommit) {

      // Assert that we really are only called for each new
      // commit:
      if (isCommit) {
        String fileName = ((IndexCommit) commits.get(commits.size()-1)).getSegmentsFileName();
        if (seen.contains(fileName)) {
          throw new RuntimeException("onCommit was called twice on the same commit point: " + fileName);
        }
        seen.add(fileName);
        numOnCommit++;
      }
      int size = commits.size();
      for(int i=0;i<size-numToKeep;i++) {
        ((IndexCommit) commits.get(i)).delete();
        numDelete++;
      }
    }
  }

  /*
   * Delete a commit only when it has been obsoleted by N
   * seconds.
   */
  class ExpirationTimeDeletionPolicy implements IndexDeletionPolicy {

    Directory dir;
    double expirationTimeSeconds;
    int numDelete;

    public ExpirationTimeDeletionPolicy(Directory dir, double seconds) {
      this.dir = dir;
      this.expirationTimeSeconds = seconds;
    }

    public void onInit(List<? extends IndexCommit> commits) throws IOException {
      verifyCommitOrder(commits);
      onCommit(commits);
    }

    public void onCommit(List<? extends IndexCommit> commits) throws IOException {
      verifyCommitOrder(commits);

      IndexCommit lastCommit = commits.get(commits.size()-1);

      // Any commit older than expireTime should be deleted:
      double expireTime = dir.fileModified(lastCommit.getSegmentsFileName())/1000.0 - expirationTimeSeconds;

      for (final IndexCommit commit : commits) {
        double modTime = dir.fileModified(commit.getSegmentsFileName())/1000.0;
        if (commit != lastCommit && modTime < expireTime) {
          commit.delete();
          numDelete += 1;
        }
      }
    }
  }

  /*
   * Test "by time expiration" deletion policy:
   */
  public void testExpirationTimeDeletionPolicy() throws IOException, InterruptedException {

    final double SECONDS = 2.0;

    Directory dir = newDirectory();
    ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS);
    IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
        new MockAnalyzer(random))
        .setIndexDeletionPolicy(policy);
    MergePolicy mp = conf.getMergePolicy();
    if (mp instanceof LogMergePolicy) {
      setUseCompoundFile(mp, true);
    }
    IndexWriter writer = new IndexWriter(dir, conf);
    writer.close();

    final int ITER = 9;

    long lastDeleteTime = 0;
    for(int i=0;i<ITER;i++) {
      // Record last time when writer performed deletes of
      // past commits
      lastDeleteTime = System.currentTimeMillis();
      conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
          new MockAnalyzer(random)).setOpenMode(
          OpenMode.APPEND).setIndexDeletionPolicy(policy);
      mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, true);
      }
      writer = new IndexWriter(dir, conf);
      for(int j=0;j<17;j++) {
        addDoc(writer);
      }
      writer.close();

      if (i < ITER-1) {
        // Make sure to sleep long enough so that some commit
        // points will be deleted:
        Thread.sleep((int) (1000.0*(SECONDS/5.0)));
      }
    }

    // First, make sure the policy in fact deleted something:
    assertTrue("no commits were deleted", policy.numDelete > 0);

    // Then simplistic check: just verify that the
    // segments_N's that still exist are in fact within SECONDS
    // seconds of the last one's mod time, and, that I can
    // open a reader on each:
    long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
    
    String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                            "",
                                                            gen);
    dir.deleteFile(IndexFileNames.SEGMENTS_GEN);

    boolean oneSecondResolution = true;

    while(gen > 0) {
      try {
        IndexReader reader = IndexReader.open(dir, true);
        reader.close();
        fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                         "",
                                                         gen);

        // if we are on a filesystem that seems to have only
        // 1 second resolution, allow +1 second in commit
        // age tolerance:
        long modTime = dir.fileModified(fileName);
        oneSecondResolution &= (modTime % 1000) == 0;
        final long leeway = (long) ((SECONDS + (oneSecondResolution ? 1.0:0.0))*1000);

        assertTrue("commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted ", lastDeleteTime - modTime <= leeway);
      } catch (IOException e) {
        // OK
        break;
      }
      
      dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
      gen--;
    }

    dir.close();
  }

  /*
   * Test a silly deletion policy that keeps all commits around.
   */
  public void testKeepAllDeletionPolicy() throws IOException {
    for(int pass=0;pass<2;pass++) {

      if (VERBOSE) {
        System.out.println("TEST: cycle pass=" + pass);
      }

      boolean useCompoundFile = (pass % 2) != 0;

      // Never deletes a commit
      KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();

      Directory dir = newDirectory();
      policy.dir = dir;

      IndexWriterConfig conf = newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setIndexDeletionPolicy(policy).setMaxBufferedDocs(10)
          .setMergeScheduler(new SerialMergeScheduler());
      MergePolicy mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
      IndexWriter writer = new IndexWriter(dir, conf);
      for(int i=0;i<107;i++) {
        addDoc(writer);
      }
      writer.close();

      final boolean isOptimized;
      {
        IndexReader r = IndexReader.open(dir);
        isOptimized = r.isOptimized();
        r.close();
      }
      if (!isOptimized) {
        conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
                                    new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(
                                                                                              OpenMode.APPEND).setIndexDeletionPolicy(policy);
        mp = conf.getMergePolicy();
        if (mp instanceof LogMergePolicy) {
          setUseCompoundFile(mp, true);
        }
        if (VERBOSE) {
          System.out.println("TEST: open writer for optimize");
        }
        writer = new IndexWriter(dir, conf);
        writer.setInfoStream(VERBOSE ? System.out : null);
        writer.optimize();
        writer.close();
      }
      assertEquals(isOptimized ? 0:1, policy.numOnInit);

      // If we are not auto committing then there should
      // be exactly 2 commits (one per close above):
      assertEquals(1 + (isOptimized ? 0:1), policy.numOnCommit);

      // Test listCommits
      Collection<IndexCommit> commits = IndexReader.listCommits(dir);
      // 2 from closing writer
      assertEquals(1 + (isOptimized ? 0:1), commits.size());

      // Make sure we can open a reader on each commit:
      for (final IndexCommit commit : commits) {
        IndexReader r = IndexReader.open(commit, null, false);
        r.close();
      }

      // Simplistic check: just verify all segments_N's still
      // exist, and, I can open a reader on each:
      dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
      while(gen > 0) {
        IndexReader reader = IndexReader.open(dir, true);
        reader.close();
        dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        gen--;

        if (gen > 0) {
          // Now that we've removed a commit point, which
          // should have orphan'd at least one index file.
          // Open & close a writer and assert that it
          // actually removed something:
          int preCount = dir.listAll().length;
          writer = new IndexWriter(dir, newIndexWriterConfig(
              TEST_VERSION_CURRENT,
              new MockAnalyzer(random)).setOpenMode(
              OpenMode.APPEND).setIndexDeletionPolicy(policy));
          writer.close();
          int postCount = dir.listAll().length;
          assertTrue(postCount < preCount);
        }
      }

      dir.close();
    }
  }

  /* Uses KeepAllDeletionPolicy to keep all commits around,
   * then, opens a new IndexWriter on a previous commit
   * point. */
  public void testOpenPriorSnapshot() throws IOException {
    // Never deletes a commit
    KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();

    Directory dir = newDirectory();
    policy.dir = dir;

    IndexWriter writer = new IndexWriter(
        dir,
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
            setIndexDeletionPolicy(policy).
            setMaxBufferedDocs(2).
            setMergePolicy(newLogMergePolicy(10))
    );
    for(int i=0;i<10;i++) {
      addDoc(writer);
      if ((1+i)%2 == 0)
        writer.commit();
    }
    writer.close();

    Collection<IndexCommit> commits = IndexReader.listCommits(dir);
    assertEquals(5, commits.size());
    IndexCommit lastCommit = null;
    for (final IndexCommit commit : commits) {
      if (lastCommit == null || commit.getGeneration() > lastCommit.getGeneration())
        lastCommit = commit;
    }
    assertTrue(lastCommit != null);

    // Now add 1 doc and optimize
    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
        new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
    addDoc(writer);
    assertEquals(11, writer.numDocs());
    writer.optimize();
    writer.close();

    assertEquals(6, IndexReader.listCommits(dir).size());

    // Now open writer on the commit just before optimize:
    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
        .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
    assertEquals(10, writer.numDocs());

    // Should undo our rollback:
    writer.rollback();

    IndexReader r = IndexReader.open(dir, true);
    // Still optimized, still 11 docs
    assertTrue(r.isOptimized());
    assertEquals(11, r.numDocs());
    r.close();

    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
        .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
    assertEquals(10, writer.numDocs());
    // Commits the rollback:
    writer.close();

    // Now 8 because we made another commit
    assertEquals(7, IndexReader.listCommits(dir).size());
    
    r = IndexReader.open(dir, true);
    // Not optimized because we rolled it back, and now only
    // 10 docs
    assertTrue(!r.isOptimized());
    assertEquals(10, r.numDocs());
    r.close();

    // Reoptimize
    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
    writer.optimize();
    writer.close();

    r = IndexReader.open(dir, true);
    assertTrue(r.isOptimized());
    assertEquals(10, r.numDocs());
    r.close();

    // Now open writer on the commit just before optimize,
    // but this time keeping only the last commit:
    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexCommit(lastCommit));
    assertEquals(10, writer.numDocs());
    
    // Reader still sees optimized index, because writer
    // opened on the prior commit has not yet committed:
    r = IndexReader.open(dir, true);
    assertTrue(r.isOptimized());
    assertEquals(10, r.numDocs());
    r.close();

    writer.close();

    // Now reader sees unoptimized index:
    r = IndexReader.open(dir, true);
    assertTrue(!r.isOptimized());
    assertEquals(10, r.numDocs());
    r.close();

    dir.close();
  }


  /* Test keeping NO commit points.  This is a viable and
   * useful case eg where you want to build a big index and
   * you know there are no readers.
   */
  public void testKeepNoneOnInitDeletionPolicy() throws IOException {
    for(int pass=0;pass<2;pass++) {

      boolean useCompoundFile = (pass % 2) != 0;

      KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy();

      Directory dir = newDirectory();

      IndexWriterConfig conf = newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
          .setMaxBufferedDocs(10);
      MergePolicy mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
      IndexWriter writer = new IndexWriter(dir, conf);
      for(int i=0;i<107;i++) {
        addDoc(writer);
      }
      writer.close();

      conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
      mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, true);
      }
      writer = new IndexWriter(dir, conf);
      writer.optimize();
      writer.close();

      assertEquals(1, policy.numOnInit);
      // If we are not auto committing then there should
      // be exactly 2 commits (one per close above):
      assertEquals(2, policy.numOnCommit);

      // Simplistic check: just verify the index is in fact
      // readable:
      IndexReader reader = IndexReader.open(dir, true);
      reader.close();

      dir.close();
    }
  }

  /*
   * Test a deletion policy that keeps last N commits.
   */
  public void testKeepLastNDeletionPolicy() throws IOException {
    final int N = 5;

    for(int pass=0;pass<2;pass++) {

      boolean useCompoundFile = (pass % 2) != 0;

      Directory dir = newDirectory();

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);

      for(int j=0;j<N+1;j++) {
        IndexWriterConfig conf = newIndexWriterConfig(
            TEST_VERSION_CURRENT, new MockAnalyzer(random))
            .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
            .setMaxBufferedDocs(10);
        MergePolicy mp = conf.getMergePolicy();
        if (mp instanceof LogMergePolicy) {
          setUseCompoundFile(mp, useCompoundFile);
        }
        IndexWriter writer = new IndexWriter(dir, conf);
        for(int i=0;i<17;i++) {
          addDoc(writer);
        }
        writer.optimize();
        writer.close();
      }

      assertTrue(policy.numDelete > 0);
      assertEquals(N, policy.numOnInit);
      assertEquals(N+1, policy.numOnCommit);

      // Simplistic check: just verify only the past N segments_N's still
      // exist, and, I can open a reader on each:
      dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
      for(int i=0;i<N+1;i++) {
        try {
          IndexReader reader = IndexReader.open(dir, true);
          reader.close();
          if (i == N) {
            fail("should have failed on commits prior to last " + N);
          }
        } catch (IOException e) {
          if (i != N) {
            throw e;
          }
        }
        if (i < N) {
          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        }
        gen--;
      }

      dir.close();
    }
  }

  /*
   * Test a deletion policy that keeps last N commits
   * around, with reader doing deletes.
   */
  public void testKeepLastNDeletionPolicyWithReader() throws IOException {
    final int N = 10;

    for(int pass=0;pass<2;pass++) {

      boolean useCompoundFile = (pass % 2) != 0;

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);

      Directory dir = newDirectory();
      IndexWriterConfig conf = newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random))
        .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
      MergePolicy mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
      IndexWriter writer = new IndexWriter(dir, conf);
      writer.close();
      Term searchTerm = new Term("content", "aaa");        
      Query query = new TermQuery(searchTerm);

      for(int i=0;i<N+1;i++) {
        if (VERBOSE) {
          System.out.println("\nTEST: cycle i=" + i);
        }
        conf = newIndexWriterConfig(
            TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
        mp = conf.getMergePolicy();
        if (mp instanceof LogMergePolicy) {
          setUseCompoundFile(mp, useCompoundFile);
        }
        writer = new IndexWriter(dir, conf);
        writer.setInfoStream(VERBOSE ? System.out : null);
        for(int j=0;j<17;j++) {
          addDoc(writer);
        }
        // this is a commit
        if (VERBOSE) {
          System.out.println("TEST: close writer");
        }
        writer.close();
        IndexReader reader = IndexReader.open(dir, policy, false);
        reader.deleteDocument(3*i+1);
        reader.setNorm(4*i+1, "content", 2.0F);
        IndexSearcher searcher = newSearcher(reader);
        ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
        assertEquals(16*(1+i), hits.length);
        // this is a commit
        if (VERBOSE) {
          System.out.println("TEST: close reader numOnCommit=" + policy.numOnCommit);
        }
        reader.close();
        searcher.close();
      }
      conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
      mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
      IndexReader r = IndexReader.open(dir);
      final boolean wasOptimized = r.isOptimized();
      r.close();
      writer = new IndexWriter(dir, conf);
      writer.optimize();
      // this is a commit
      writer.close();

      assertEquals(2*(N+1)+1, policy.numOnInit);
      assertEquals(2*(N+2) - (wasOptimized ? 1:0), policy.numOnCommit);

      IndexSearcher searcher = new IndexSearcher(dir, false);
      ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
      assertEquals(176, hits.length);

      // Simplistic check: just verify only the past N segments_N's still
      // exist, and, I can open a reader on each:
      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);

      dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
      int expectedCount = 176;
      searcher.close();
      for(int i=0;i<N+1;i++) {
        try {
          IndexReader reader = IndexReader.open(dir, true);

          // Work backwards in commits on what the expected
          // count should be.
          searcher = newSearcher(reader);
          hits = searcher.search(query, null, 1000).scoreDocs;
          if (i > 1) {
            if (i % 2 == 0) {
              expectedCount += 1;
            } else {
              expectedCount -= 17;
            }
          }
          assertEquals(expectedCount, hits.length);
          searcher.close();
          reader.close();
          if (i == N) {
            fail("should have failed on commits before last 5");
          }
        } catch (IOException e) {
          if (i != N) {
            throw e;
          }
        }
        if (i < N) {
          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        }
        gen--;
      }
      dir.close();
    }
  }

  /*
   * Test a deletion policy that keeps last N commits
   * around, through creates.
   */
  public void testKeepLastNDeletionPolicyWithCreates() throws IOException {
    
    final int N = 10;

    for(int pass=0;pass<2;pass++) {

      boolean useCompoundFile = (pass % 2) != 0;

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);

      Directory dir = newDirectory();
      IndexWriterConfig conf = newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
          .setMaxBufferedDocs(10);
      MergePolicy mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
      IndexWriter writer = new IndexWriter(dir, conf);
      writer.close();
      Term searchTerm = new Term("content", "aaa");        
      Query query = new TermQuery(searchTerm);

      for(int i=0;i<N+1;i++) {

        conf = newIndexWriterConfig(
            TEST_VERSION_CURRENT, new MockAnalyzer(random))
            .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy)
            .setMaxBufferedDocs(10);
        mp = conf.getMergePolicy();
        if (mp instanceof LogMergePolicy) {
          setUseCompoundFile(mp, useCompoundFile);
        }
        writer = new IndexWriter(dir, conf);
        for(int j=0;j<17;j++) {
          addDoc(writer);
        }
        // this is a commit
        writer.close();
        IndexReader reader = IndexReader.open(dir, policy, false);
        reader.deleteDocument(3);
        reader.setNorm(5, "content", 2.0F);
        IndexSearcher searcher = newSearcher(reader);
        ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
        assertEquals(16, hits.length);
        // this is a commit
        reader.close();
        searcher.close();

        writer = new IndexWriter(dir, newIndexWriterConfig(
            TEST_VERSION_CURRENT, new MockAnalyzer(random))
            .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy));
        // This will not commit: there are no changes
        // pending because we opened for "create":
        writer.close();
      }

      assertEquals(3*(N+1), policy.numOnInit);
      assertEquals(3*(N+1)+1, policy.numOnCommit);

      IndexSearcher searcher = new IndexSearcher(dir, false);
      ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
      assertEquals(0, hits.length);

      // Simplistic check: just verify only the past N segments_N's still
      // exist, and, I can open a reader on each:
      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);

      dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
      int expectedCount = 0;

      for(int i=0;i<N+1;i++) {
        try {
          IndexReader reader = IndexReader.open(dir, true);

          // Work backwards in commits on what the expected
          // count should be.
          searcher = newSearcher(reader);
          hits = searcher.search(query, null, 1000).scoreDocs;
          assertEquals(expectedCount, hits.length);
          searcher.close();
          if (expectedCount == 0) {
            expectedCount = 16;
          } else if (expectedCount == 16) {
            expectedCount = 17;
          } else if (expectedCount == 17) {
            expectedCount = 0;
          }
          reader.close();
          if (i == N) {
            fail("should have failed on commits before last " + N);
          }
        } catch (IOException e) {
          if (i != N) {
            throw e;
          }
        }
        if (i < N) {
          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        }
        gen--;
      }
      
      dir.close();
    }
  }

  private void addDoc(IndexWriter writer) throws IOException
  {
    Document doc = new Document();
    doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
    writer.addDocument(doc);
  }
}

Other Lucene examples (source code examples)

Here is a short list of links related to this Lucene TestDeletionPolicy.java source code file:



my book on functional programming

 

new blog posts

 

Copyright 1998-2019 Alvin Alexander, alvinalexander.com
All Rights Reserved.

A percentage of advertising revenue from
pages under the /java/jwarehouse URI on this website is
paid back to open source projects.