|
|
Lucene example source code file (DirectoryReader.java)
This example Lucene source code file (DirectoryReader.java) is included in the DevDaily.com
"Java Source Code
Warehouse" project. The intent of this project is to help you "Learn Java by Example" TM.
The Lucene DirectoryReader.java source code
package org.apache.lucene.index;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.MapBackedSet;
/**
* An IndexReader which reads indexes with multiple segments.
*/
class DirectoryReader extends IndexReader implements Cloneable {
protected Directory directory;
protected boolean readOnly;
IndexWriter writer;
private IndexDeletionPolicy deletionPolicy;
private Lock writeLock;
private final SegmentInfos segmentInfos;
private boolean stale;
private final int termInfosIndexDivisor;
private boolean rollbackHasChanges;
private SegmentReader[] subReaders;
private int[] starts; // 1st docno for each segment
private Map<String,byte[]> normsCache = new HashMap();
private int maxDoc = 0;
private int numDocs = -1;
private boolean hasDeletions = false;
// Max version in index as of when we opened; this can be
// > our current segmentInfos version in case we were
// opened on a past IndexCommit:
private long maxIndexVersion;
private final boolean applyAllDeletes;
static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
@Override
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
SegmentInfos infos = new SegmentInfos();
infos.read(directory, segmentFileName);
if (readOnly)
return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, null);
else
return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, null);
}
}.run(commit);
}
/** Construct reading the named set of readers. */
DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor,
Collection<ReaderFinishedListener> readerFinishedListeners) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = sis;
this.deletionPolicy = deletionPolicy;
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (readerFinishedListeners == null) {
this.readerFinishedListeners = new MapBackedSet<ReaderFinishedListener>(new ConcurrentHashMap());
} else {
this.readerFinishedListeners = readerFinishedListeners;
}
applyAllDeletes = false;
// To reduce the chance of hitting FileNotFound
// (and having to retry), we open segments in
// reverse because IndexWriter merges & deletes
// the newest segments first.
SegmentReader[] readers = new SegmentReader[sis.size()];
for (int i = sis.size()-1; i >= 0; i--) {
boolean success = false;
try {
readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor);
readers[i].readerFinishedListeners = this.readerFinishedListeners;
success = true;
} finally {
if (!success) {
// Close all readers we had opened:
for(i++;i<sis.size();i++) {
try {
readers[i].close();
} catch (Throwable ignore) {
// keep going - we want to clean up as much as possible
}
}
}
}
}
initialize(readers);
}
// Used by near real-time search
DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, boolean applyAllDeletes) throws IOException {
this.directory = writer.getDirectory();
this.readOnly = true;
this.applyAllDeletes = applyAllDeletes; // saved for reopen
this.termInfosIndexDivisor = termInfosIndexDivisor;
readerFinishedListeners = writer.getReaderFinishedListeners();
// IndexWriter synchronizes externally before calling
// us, which ensures infos will not change; so there's
// no need to process segments in reverse order
final int numSegments = infos.size();
List<SegmentReader> readers = new ArrayList();
final Directory dir = writer.getDirectory();
segmentInfos = (SegmentInfos) infos.clone();
int infosUpto = 0;
for (int i=0;i<numSegments;i++) {
boolean success = false;
try {
final SegmentInfo info = infos.info(i);
assert info.dir == dir;
final SegmentReader reader = writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor);
if (reader.numDocs() > 0 || writer.getKeepFullyDeletedSegments()) {
reader.readerFinishedListeners = readerFinishedListeners;
readers.add(reader);
infosUpto++;
} else {
reader.close();
segmentInfos.remove(infosUpto);
}
success = true;
} finally {
if (!success) {
// Close all readers we had opened:
for(SegmentReader reader : readers) {
try {
reader.close();
} catch (Throwable ignore) {
// keep going - we want to clean up as much as possible
}
}
}
}
}
this.writer = writer;
initialize(readers.toArray(new SegmentReader[readers.size()]));
}
/** This constructor is only used for {@link #reopen()} */
DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,
Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor,
Collection<ReaderFinishedListener> readerFinishedListeners) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = infos;
this.termInfosIndexDivisor = termInfosIndexDivisor;
assert readerFinishedListeners != null;
this.readerFinishedListeners = readerFinishedListeners;
applyAllDeletes = false;
// we put the old SegmentReaders in a map, that allows us
// to lookup a reader using its segment name
Map<String,Integer> segmentReaders = new HashMap();
if (oldReaders != null) {
// create a Map SegmentName->SegmentReader
for (int i = 0; i < oldReaders.length; i++) {
segmentReaders.put(oldReaders[i].getSegmentName(), Integer.valueOf(i));
}
}
SegmentReader[] newReaders = new SegmentReader[infos.size()];
// remember which readers are shared between the old and the re-opened
// DirectoryReader - we have to incRef those readers
boolean[] readerShared = new boolean[infos.size()];
for (int i = infos.size() - 1; i>=0; i--) {
// find SegmentReader for this segment
Integer oldReaderIndex = segmentReaders.get(infos.info(i).name);
if (oldReaderIndex == null) {
// this is a new segment, no old SegmentReader can be reused
newReaders[i] = null;
} else {
// there is an old reader for this segment - we'll try to reopen it
newReaders[i] = oldReaders[oldReaderIndex.intValue()];
}
boolean success = false;
try {
SegmentReader newReader;
if (newReaders[i] == null || infos.info(i).getUseCompoundFile() != newReaders[i].getSegmentInfo().getUseCompoundFile()) {
// We should never see a totally new segment during cloning
assert !doClone;
// this is a new reader; in case we hit an exception we can close it safely
newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor);
newReader.readerFinishedListeners = readerFinishedListeners;
} else {
newReader = newReaders[i].reopenSegment(infos.info(i), doClone, readOnly);
assert newReader.readerFinishedListeners == readerFinishedListeners;
}
if (newReader == newReaders[i]) {
// this reader will be shared between the old and the new one,
// so we must incRef it
readerShared[i] = true;
newReader.incRef();
} else {
readerShared[i] = false;
newReaders[i] = newReader;
}
success = true;
} finally {
if (!success) {
for (i++; i < infos.size(); i++) {
if (newReaders[i] != null) {
try {
if (!readerShared[i]) {
// this is a new subReader that is not used by the old one,
// we can close it
newReaders[i].close();
} else {
// this subReader is also used by the old reader, so instead
// closing we must decRef it
newReaders[i].decRef();
}
} catch (IOException ignore) {
// keep going - we want to clean up as much as possible
}
}
}
}
}
}
// initialize the readers to calculate maxDoc before we try to reuse the old normsCache
initialize(newReaders);
// try to copy unchanged norms from the old normsCache to the new one
if (oldNormsCache != null) {
for (Map.Entry<String,byte[]> entry: oldNormsCache.entrySet()) {
String field = entry.getKey();
if (!hasNorms(field)) {
continue;
}
byte[] oldBytes = entry.getValue();
byte[] bytes = new byte[maxDoc()];
for (int i = 0; i < subReaders.length; i++) {
Integer oldReaderIndex = segmentReaders.get(subReaders[i].getSegmentName());
// this SegmentReader was not re-opened, we can copy all of its norms
if (oldReaderIndex != null &&
(oldReaders[oldReaderIndex.intValue()] == subReaders[i]
|| oldReaders[oldReaderIndex.intValue()].norms.get(field) == subReaders[i].norms.get(field))) {
// we don't have to synchronize here: either this constructor is called from a SegmentReader,
// in which case no old norms cache is present, or it is called from MultiReader.reopen(),
// which is synchronized
System.arraycopy(oldBytes, oldStarts[oldReaderIndex.intValue()], bytes, starts[i], starts[i+1] - starts[i]);
} else {
subReaders[i].norms(field, bytes, starts[i]);
}
}
normsCache.put(field, bytes); // update cache
}
}
}
/** {@inheritDoc} */
@Override
public String toString() {
final StringBuilder buffer = new StringBuilder();
if (hasChanges) {
buffer.append("*");
}
buffer.append(getClass().getSimpleName());
buffer.append('(');
final String segmentsFile = segmentInfos.getCurrentSegmentFileName();
if (segmentsFile != null) {
buffer.append(segmentsFile);
}
if (writer != null) {
buffer.append(":nrt");
}
for(int i=0;i<subReaders.length;i++) {
buffer.append(' ');
buffer.append(subReaders[i]);
}
buffer.append(')');
return buffer.toString();
}
private void initialize(SegmentReader[] subReaders) throws IOException {
this.subReaders = subReaders;
starts = new int[subReaders.length + 1]; // build starts array
for (int i = 0; i < subReaders.length; i++) {
starts[i] = maxDoc;
maxDoc += subReaders[i].maxDoc(); // compute maxDocs
if (subReaders[i].hasDeletions())
hasDeletions = true;
}
starts[subReaders.length] = maxDoc;
if (!readOnly) {
maxIndexVersion = SegmentInfos.readCurrentVersion(directory);
}
}
@Override
public final synchronized Object clone() {
try {
return clone(readOnly); // Preserve current readOnly
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
@Override
public final synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException {
DirectoryReader newReader = doReopen((SegmentInfos) segmentInfos.clone(), true, openReadOnly);
if (this != newReader) {
newReader.deletionPolicy = deletionPolicy;
}
newReader.writer = writer;
// If we're cloning a non-readOnly reader, move the
// writeLock (if there is one) to the new reader:
if (!openReadOnly && writeLock != null) {
// In near real-time search, reader is always readonly
assert writer == null;
newReader.writeLock = writeLock;
newReader.hasChanges = hasChanges;
newReader.hasDeletions = hasDeletions;
writeLock = null;
hasChanges = false;
}
assert newReader.readerFinishedListeners != null;
return newReader;
}
@Override
public final IndexReader reopen() throws CorruptIndexException, IOException {
// Preserve current readOnly
return doReopen(readOnly, null);
}
@Override
public final IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException {
return doReopen(openReadOnly, null);
}
@Override
public final IndexReader reopen(final IndexCommit commit) throws CorruptIndexException, IOException {
return doReopen(true, commit);
}
private final IndexReader doReopenFromWriter(boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
assert readOnly;
if (!openReadOnly) {
throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)");
}
if (commit != null) {
throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
}
// TODO: right now we *always* make a new reader; in
// the future we could have write make some effort to
// detect that no changes have occurred
IndexReader reader = writer.getReader(applyAllDeletes);
reader.readerFinishedListeners = readerFinishedListeners;
return reader;
}
private IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
ensureOpen();
assert commit == null || openReadOnly;
// If we were obtained by writer.getReader(), re-ask the
// writer to get a new reader.
if (writer != null) {
return doReopenFromWriter(openReadOnly, commit);
} else {
return doReopenNoWriter(openReadOnly, commit);
}
}
private synchronized IndexReader doReopenNoWriter(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
if (commit == null) {
if (hasChanges) {
// We have changes, which means we are not readOnly:
assert readOnly == false;
// and we hold the write lock:
assert writeLock != null;
// so no other writer holds the write lock, which
// means no changes could have been done to the index:
assert isCurrent();
if (openReadOnly) {
return clone(openReadOnly);
} else {
return this;
}
} else if (isCurrent()) {
if (openReadOnly != readOnly) {
// Just fallback to clone
return clone(openReadOnly);
} else {
return this;
}
}
} else {
if (directory != commit.getDirectory())
throw new IOException("the specified commit does not match the specified Directory");
if (segmentInfos != null && commit.getSegmentsFileName().equals(segmentInfos.getCurrentSegmentFileName())) {
if (readOnly != openReadOnly) {
// Just fallback to clone
return clone(openReadOnly);
} else {
return this;
}
}
}
return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
@Override
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
SegmentInfos infos = new SegmentInfos();
infos.read(directory, segmentFileName);
return doReopen(infos, false, openReadOnly);
}
}.run(commit);
}
private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
DirectoryReader reader;
if (openReadOnly) {
reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, readerFinishedListeners);
} else {
reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, readerFinishedListeners);
}
return reader;
}
/** Version number when this IndexReader was opened. */
@Override
public long getVersion() {
ensureOpen();
return segmentInfos.getVersion();
}
@Override
public TermFreqVector[] getTermFreqVectors(int n) throws IOException {
ensureOpen();
int i = readerIndex(n); // find segment num
return subReaders[i].getTermFreqVectors(n - starts[i]); // dispatch to segment
}
@Override
public TermFreqVector getTermFreqVector(int n, String field)
throws IOException {
ensureOpen();
int i = readerIndex(n); // find segment num
return subReaders[i].getTermFreqVector(n - starts[i], field);
}
@Override
public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) throws IOException {
ensureOpen();
int i = readerIndex(docNumber); // find segment num
subReaders[i].getTermFreqVector(docNumber - starts[i], field, mapper);
}
@Override
public void getTermFreqVector(int docNumber, TermVectorMapper mapper) throws IOException {
ensureOpen();
int i = readerIndex(docNumber); // find segment num
subReaders[i].getTermFreqVector(docNumber - starts[i], mapper);
}
/**
* Checks is the index is optimized (if it has a single segment and no deletions)
* @return <code>true if the index is optimized; false otherwise
*/
@Override
public boolean isOptimized() {
ensureOpen();
return segmentInfos.size() == 1 && !hasDeletions();
}
@Override
public int numDocs() {
// Don't call ensureOpen() here (it could affect performance)
// NOTE: multiple threads may wind up init'ing
// numDocs... but that's harmless
if (numDocs == -1) { // check cache
int n = 0; // cache miss--recompute
for (int i = 0; i < subReaders.length; i++)
n += subReaders[i].numDocs(); // sum from readers
numDocs = n;
}
return numDocs;
}
@Override
public int maxDoc() {
// Don't call ensureOpen() here (it could affect performance)
return maxDoc;
}
// inherit javadoc
@Override
public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
ensureOpen();
int i = readerIndex(n); // find segment num
return subReaders[i].document(n - starts[i], fieldSelector); // dispatch to segment reader
}
@Override
public boolean isDeleted(int n) {
// Don't call ensureOpen() here (it could affect performance)
final int i = readerIndex(n); // find segment num
return subReaders[i].isDeleted(n - starts[i]); // dispatch to segment reader
}
@Override
public boolean hasDeletions() {
// Don't call ensureOpen() here (it could affect performance)
return hasDeletions;
}
@Override
protected void doDelete(int n) throws CorruptIndexException, IOException {
numDocs = -1; // invalidate cache
int i = readerIndex(n); // find segment num
subReaders[i].deleteDocument(n - starts[i]); // dispatch to segment reader
hasDeletions = true;
}
@Override
protected void doUndeleteAll() throws CorruptIndexException, IOException {
for (int i = 0; i < subReaders.length; i++)
subReaders[i].undeleteAll();
hasDeletions = false;
numDocs = -1; // invalidate cache
}
private int readerIndex(int n) { // find reader for doc n:
return readerIndex(n, this.starts, this.subReaders.length);
}
final static int readerIndex(int n, int[] starts, int numSubReaders) { // find reader for doc n:
int lo = 0; // search starts array
int hi = numSubReaders - 1; // for first element less
while (hi >= lo) {
int mid = (lo + hi) >>> 1;
int midValue = starts[mid];
if (n < midValue)
hi = mid - 1;
else if (n > midValue)
lo = mid + 1;
else { // found a match
while (mid+1 < numSubReaders && starts[mid+1] == midValue) {
mid++; // scan to last match
}
return mid;
}
}
return hi;
}
@Override
public boolean hasNorms(String field) throws IOException {
ensureOpen();
for (int i = 0; i < subReaders.length; i++) {
if (subReaders[i].hasNorms(field)) return true;
}
return false;
}
@Override
public synchronized byte[] norms(String field) throws IOException {
ensureOpen();
byte[] bytes = normsCache.get(field);
if (bytes != null)
return bytes; // cache hit
if (!hasNorms(field))
return null;
bytes = new byte[maxDoc()];
for (int i = 0; i < subReaders.length; i++)
subReaders[i].norms(field, bytes, starts[i]);
normsCache.put(field, bytes); // update cache
return bytes;
}
@Override
public synchronized void norms(String field, byte[] result, int offset)
throws IOException {
ensureOpen();
byte[] bytes = normsCache.get(field);
if (bytes==null && !hasNorms(field)) {
Arrays.fill(result, offset, result.length, Similarity.getDefault().encodeNormValue(1.0f));
} else if (bytes != null) { // cache hit
System.arraycopy(bytes, 0, result, offset, maxDoc());
} else {
for (int i = 0; i < subReaders.length; i++) { // read from segments
subReaders[i].norms(field, result, offset + starts[i]);
}
}
}
@Override
protected void doSetNorm(int n, String field, byte value)
throws CorruptIndexException, IOException {
synchronized (normsCache) {
normsCache.remove(field); // clear cache
}
int i = readerIndex(n); // find segment num
subReaders[i].setNorm(n-starts[i], field, value); // dispatch
}
@Override
public TermEnum terms() throws IOException {
ensureOpen();
if (subReaders.length == 1) {
// Optimize single segment case:
return subReaders[0].terms();
} else {
return new MultiTermEnum(this, subReaders, starts, null);
}
}
@Override
public TermEnum terms(Term term) throws IOException {
ensureOpen();
if (subReaders.length == 1) {
// Optimize single segment case:
return subReaders[0].terms(term);
} else {
return new MultiTermEnum(this, subReaders, starts, term);
}
}
@Override
public int docFreq(Term t) throws IOException {
ensureOpen();
int total = 0; // sum freqs in segments
for (int i = 0; i < subReaders.length; i++)
total += subReaders[i].docFreq(t);
return total;
}
@Override
public TermDocs termDocs() throws IOException {
ensureOpen();
if (subReaders.length == 1) {
// Optimize single segment case:
return subReaders[0].termDocs();
} else {
return new MultiTermDocs(this, subReaders, starts);
}
}
@Override
public TermDocs termDocs(Term term) throws IOException {
ensureOpen();
if (subReaders.length == 1) {
// Optimize single segment case:
return subReaders[0].termDocs(term);
} else {
return super.termDocs(term);
}
}
@Override
public TermPositions termPositions() throws IOException {
ensureOpen();
if (subReaders.length == 1) {
// Optimize single segment case:
return subReaders[0].termPositions();
} else {
return new MultiTermPositions(this, subReaders, starts);
}
}
/**
* Tries to acquire the WriteLock on this directory. this method is only valid if this IndexReader is directory
* owner.
*
* @throws StaleReaderException if the index has changed since this reader was opened
* @throws CorruptIndexException if the index is corrupt
* @throws org.apache.lucene.store.LockObtainFailedException
* if another writer has this index open (<code>write.lock could not be
* obtained)
* @throws IOException if there is a low-level IO error
*/
@Override
protected void acquireWriteLock() throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
if (readOnly) {
// NOTE: we should not reach this code w/ the core
// IndexReader classes; however, an external subclass
// of IndexReader could reach this.
ReadOnlySegmentReader.noWrite();
}
if (segmentInfos != null) {
ensureOpen();
if (stale)
throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
if (writeLock == null) {
Lock writeLock = directory.makeLock(IndexWriter.WRITE_LOCK_NAME);
if (!writeLock.obtain(IndexWriterConfig.WRITE_LOCK_TIMEOUT)) // obtain write lock
throw new LockObtainFailedException("Index locked for write: " + writeLock);
this.writeLock = writeLock;
// we have to check whether index has changed since this reader was opened.
// if so, this reader is no longer valid for
// deletion
if (SegmentInfos.readCurrentVersion(directory) > maxIndexVersion) {
stale = true;
this.writeLock.release();
this.writeLock = null;
throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
}
}
}
}
/**
* Commit changes resulting from delete, undeleteAll, or setNorm operations
* <p/>
* If an exception is hit, then either no changes or all changes will have been committed to the index (transactional
* semantics).
*
* @throws IOException if there is a low-level IO error
*/
@Override
protected void doCommit(Map<String,String> commitUserData) throws IOException {
if (hasChanges) {
segmentInfos.setUserData(commitUserData);
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
IndexFileDeleter deleter = new IndexFileDeleter(directory,
deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
segmentInfos, null);
segmentInfos.updateGeneration(deleter.getLastSegmentInfos());
segmentInfos.changed();
// Checkpoint the state we are about to change, in
// case we have to roll back:
startCommit();
final List<SegmentInfo> rollbackSegments = segmentInfos.createBackupSegmentInfos(false);
boolean success = false;
try {
for (int i = 0; i < subReaders.length; i++)
subReaders[i].commit();
// Remove segments that contain only 100% deleted
// docs:
segmentInfos.pruneDeletedSegments();
// Sync all files we just wrote
directory.sync(segmentInfos.files(directory, false));
segmentInfos.commit(directory);
success = true;
} finally {
if (!success) {
// Rollback changes that were made to
// SegmentInfos but failed to get [fully]
// committed. This way this reader instance
// remains consistent (matched to what's
// actually in the index):
rollbackCommit();
// Recompute deletable files & remove them (so
// partially written .del files, etc, are
// removed):
deleter.refresh();
// Restore all SegmentInfos (in case we pruned some)
segmentInfos.rollbackSegmentInfos(rollbackSegments);
}
}
// Have the deleter remove any now unreferenced
// files due to this commit:
deleter.checkpoint(segmentInfos, true);
deleter.close();
maxIndexVersion = segmentInfos.getVersion();
if (writeLock != null) {
writeLock.release(); // release write lock
writeLock = null;
}
}
hasChanges = false;
}
void startCommit() {
rollbackHasChanges = hasChanges;
for (int i = 0; i < subReaders.length; i++) {
subReaders[i].startCommit();
}
}
void rollbackCommit() {
hasChanges = rollbackHasChanges;
for (int i = 0; i < subReaders.length; i++) {
subReaders[i].rollbackCommit();
}
}
@Override
public Map<String,String> getCommitUserData() {
ensureOpen();
return segmentInfos.getUserData();
}
@Override
public boolean isCurrent() throws CorruptIndexException, IOException {
ensureOpen();
if (writer == null || writer.isClosed()) {
// we loaded SegmentInfos from the directory
return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion();
} else {
return writer.nrtIsCurrent(segmentInfos);
}
}
@Override
protected synchronized void doClose() throws IOException {
IOException ioe = null;
normsCache = null;
for (int i = 0; i < subReaders.length; i++) {
// try to close each reader, even if an exception is thrown
try {
subReaders[i].decRef();
} catch (IOException e) {
if (ioe == null) ioe = e;
}
}
if (writer != null) {
// Since we just closed, writer may now be able to
// delete unused files:
writer.deleteUnusedFiles();
}
// throw the first exception
if (ioe != null) throw ioe;
}
@Override
public Collection<String> getFieldNames (IndexReader.FieldOption fieldNames) {
ensureOpen();
return getFieldNames(fieldNames, this.subReaders);
}
static Collection<String> getFieldNames (IndexReader.FieldOption fieldNames, IndexReader[] subReaders) {
// maintain a unique set of field names
Set<String> fieldSet = new HashSet();
for (IndexReader reader : subReaders) {
Collection<String> names = reader.getFieldNames(fieldNames);
fieldSet.addAll(names);
}
return fieldSet;
}
@Override
public IndexReader[] getSequentialSubReaders() {
return subReaders;
}
/** Returns the directory this index resides in. */
@Override
public Directory directory() {
// Don't ensureOpen here -- in certain cases, when a
// cloned/reopened reader needs to commit, it may call
// this method on the closed original reader
return directory;
}
@Override
public int getTermInfosIndexDivisor() {
return termInfosIndexDivisor;
}
/**
* Expert: return the IndexCommit that this reader has opened.
* <p/>
* @lucene.experimental
*/
@Override
public IndexCommit getIndexCommit() throws IOException {
return new ReaderCommit(segmentInfos, directory);
}
/** @see org.apache.lucene.index.IndexReader#listCommits */
public static Collection<IndexCommit> listCommits(Directory dir) throws IOException {
final String[] files = dir.listAll();
List<IndexCommit> commits = new ArrayList();
SegmentInfos latest = new SegmentInfos();
latest.read(dir);
final long currentGen = latest.getGeneration();
commits.add(new ReaderCommit(latest, dir));
for(int i=0;i<files.length;i++) {
final String fileName = files[i];
if (fileName.startsWith(IndexFileNames.SEGMENTS) &&
!fileName.equals(IndexFileNames.SEGMENTS_GEN) &&
SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) {
SegmentInfos sis = new SegmentInfos();
try {
// IOException allowed to throw there, in case
// segments_N is corrupt
sis.read(dir, fileName);
} catch (FileNotFoundException fnfe) {
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
sis = null;
}
if (sis != null)
commits.add(new ReaderCommit(sis, dir));
}
}
// Ensure that the commit points are sorted in ascending order.
Collections.sort(commits);
return commits;
}
private static final class ReaderCommit extends IndexCommit {
private String segmentsFileName;
Collection<String> files;
Directory dir;
long generation;
long version;
final boolean isOptimized;
final Map<String,String> userData;
ReaderCommit(SegmentInfos infos, Directory dir) throws IOException {
segmentsFileName = infos.getCurrentSegmentFileName();
this.dir = dir;
userData = infos.getUserData();
files = Collections.unmodifiableCollection(infos.files(dir, true));
version = infos.getVersion();
generation = infos.getGeneration();
isOptimized = infos.size() == 1 && !infos.info(0).hasDeletions();
}
@Override
public String toString() {
return "DirectoryReader.ReaderCommit(" + segmentsFileName + ")";
}
@Override
public boolean isOptimized() {
return isOptimized;
}
@Override
public String getSegmentsFileName() {
return segmentsFileName;
}
@Override
public Collection<String> getFileNames() {
return files;
}
@Override
public Directory getDirectory() {
return dir;
}
@Override
public long getVersion() {
return version;
}
@Override
public long getGeneration() {
return generation;
}
@Override
public boolean isDeleted() {
return false;
}
@Override
public Map<String,String> getUserData() {
return userData;
}
@Override
public void delete() {
throw new UnsupportedOperationException("This IndexCommit does not support deletions");
}
}
static class MultiTermEnum extends TermEnum {
IndexReader topReader; // used for matching TermEnum to TermDocs
private SegmentMergeQueue queue;
private Term term;
private int docFreq;
final SegmentMergeInfo[] matchingSegments; // null terminated array of matching segments
public MultiTermEnum(IndexReader topReader, IndexReader[] readers, int[] starts, Term t)
throws IOException {
this.topReader = topReader;
queue = new SegmentMergeQueue(readers.length);
matchingSegments = new SegmentMergeInfo[readers.length+1];
for (int i = 0; i < readers.length; i++) {
IndexReader reader = readers[i];
TermEnum termEnum;
if (t != null) {
termEnum = reader.terms(t);
} else
termEnum = reader.terms();
SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader);
smi.ord = i;
if (t == null ? smi.next() : termEnum.term() != null)
queue.add(smi); // initialize queue
else
smi.close();
}
if (t != null && queue.size() > 0) {
next();
}
}
@Override
public boolean next() throws IOException {
for (int i=0; i<matchingSegments.length; i++) {
SegmentMergeInfo smi = matchingSegments[i];
if (smi==null) break;
if (smi.next())
queue.add(smi);
else
smi.close(); // done with segment
}
int numMatchingSegments = 0;
matchingSegments[0] = null;
SegmentMergeInfo top = queue.top();
if (top == null) {
term = null;
return false;
}
term = top.term;
docFreq = 0;
while (top != null && term.compareTo(top.term) == 0) {
matchingSegments[numMatchingSegments++] = top;
queue.pop();
docFreq += top.termEnum.docFreq(); // increment freq
top = queue.top();
}
matchingSegments[numMatchingSegments] = null;
return true;
}
@Override
public Term term() {
return term;
}
@Override
public int docFreq() {
return docFreq;
}
@Override
public void close() throws IOException {
queue.close();
}
}
static class MultiTermDocs implements TermDocs {
IndexReader topReader; // used for matching TermEnum to TermDocs
protected IndexReader[] readers;
protected int[] starts;
protected Term term;
protected int base = 0;
protected int pointer = 0;
private TermDocs[] readerTermDocs;
protected TermDocs current; // == readerTermDocs[pointer]
private MultiTermEnum tenum; // the term enum used for seeking... can be null
int matchingSegmentPos; // position into the matching segments from tenum
SegmentMergeInfo smi; // current segment mere info... can be null
public MultiTermDocs(IndexReader topReader, IndexReader[] r, int[] s) {
this.topReader = topReader;
readers = r;
starts = s;
readerTermDocs = new TermDocs[r.length];
}
public int doc() {
return base + current.doc();
}
public int freq() {
return current.freq();
}
public void seek(Term term) {
this.term = term;
this.base = 0;
this.pointer = 0;
this.current = null;
this.tenum = null;
this.smi = null;
this.matchingSegmentPos = 0;
}
public void seek(TermEnum termEnum) throws IOException {
seek(termEnum.term());
if (termEnum instanceof MultiTermEnum) {
tenum = (MultiTermEnum)termEnum;
if (topReader != tenum.topReader)
tenum = null;
}
}
public boolean next() throws IOException {
for(;;) {
if (current!=null && current.next()) {
return true;
}
else if (pointer < readers.length) {
if (tenum != null) {
smi = tenum.matchingSegments[matchingSegmentPos++];
if (smi==null) {
pointer = readers.length;
return false;
}
pointer = smi.ord;
}
base = starts[pointer];
current = termDocs(pointer++);
} else {
return false;
}
}
}
/** Optimized implementation. */
public int read(final int[] docs, final int[] freqs) throws IOException {
while (true) {
while (current == null) {
if (pointer < readers.length) { // try next segment
if (tenum != null) {
smi = tenum.matchingSegments[matchingSegmentPos++];
if (smi==null) {
pointer = readers.length;
return 0;
}
pointer = smi.ord;
}
base = starts[pointer];
current = termDocs(pointer++);
} else {
return 0;
}
}
int end = current.read(docs, freqs);
if (end == 0) { // none left in segment
current = null;
} else { // got some
final int b = base; // adjust doc numbers
for (int i = 0; i < end; i++)
docs[i] += b;
return end;
}
}
}
/* A Possible future optimization could skip entire segments */
public boolean skipTo(int target) throws IOException {
for(;;) {
if (current != null && current.skipTo(target-base)) {
return true;
} else if (pointer < readers.length) {
if (tenum != null) {
SegmentMergeInfo smi = tenum.matchingSegments[matchingSegmentPos++];
if (smi==null) {
pointer = readers.length;
return false;
}
pointer = smi.ord;
}
base = starts[pointer];
current = termDocs(pointer++);
} else
return false;
}
}
private TermDocs termDocs(int i) throws IOException {
TermDocs result = readerTermDocs[i];
if (result == null)
result = readerTermDocs[i] = termDocs(readers[i]);
if (smi != null) {
assert(smi.ord == i);
assert(smi.termEnum.term().equals(term));
result.seek(smi.termEnum);
} else {
result.seek(term);
}
return result;
}
protected TermDocs termDocs(IndexReader reader)
throws IOException {
return term==null ? reader.termDocs(null) : reader.termDocs();
}
public void close() throws IOException {
for (int i = 0; i < readerTermDocs.length; i++) {
if (readerTermDocs[i] != null)
readerTermDocs[i].close();
}
}
}
static class MultiTermPositions extends MultiTermDocs implements TermPositions {
public MultiTermPositions(IndexReader topReader, IndexReader[] r, int[] s) {
super(topReader,r,s);
}
@Override
protected TermDocs termDocs(IndexReader reader) throws IOException {
return reader.termPositions();
}
public int nextPosition() throws IOException {
return ((TermPositions)current).nextPosition();
}
public int getPayloadLength() {
return ((TermPositions)current).getPayloadLength();
}
public byte[] getPayload(byte[] data, int offset) throws IOException {
return ((TermPositions)current).getPayload(data, offset);
}
// TODO: Remove warning after API has been finalized
public boolean isPayloadAvailable() {
return ((TermPositions) current).isPayloadAvailable();
}
}
}
Other Lucene examples (source code examples)
Here is a short list of links related to this Lucene DirectoryReader.java source code file:
|
The search page
Other Lucene source code examples at this package level
Click here to learn more about this project
