Lucene example source code file (IndexTimeSynonymTest.java)

This example Lucene source code file (IndexTimeSynonymTest.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Java - Lucene tags/keywords

booleanquery, exception, exception, f, f, fieldphraselist, fieldquery, fieldquery, fieldtermstack, fieldtermstack, io, mac, my, set, token, util

The Lucene IndexTimeSynonymTest.java source code

package org.apache.lucene.search.vectorhighlight;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.io.Reader;
import java.util.HashSet;
import java.util.Set;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.AttributeImpl;

public class IndexTimeSynonymTest extends AbstractTestCase {
  
  public void testFieldTermStackIndex1wSearch1term() throws Exception {
    makeIndex1w();
    
    FieldQuery fq = new FieldQuery( tq( "Mac" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 1, stack.termList.size() );
    assertEquals( "Mac(11,20,3)", stack.pop().toString() );
  }
  
  public void testFieldTermStackIndex1wSearch2terms() throws Exception {
    makeIndex1w();

    BooleanQuery bq = new BooleanQuery();
    bq.add( tq( "Mac" ), Occur.SHOULD );
    bq.add( tq( "MacBook" ), Occur.SHOULD );
    FieldQuery fq = new FieldQuery( bq, true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 2, stack.termList.size() );
    Set<String> expectedSet = new HashSet();
    expectedSet.add( "Mac(11,20,3)" );
    expectedSet.add( "MacBook(11,20,3)" );
    assertTrue( expectedSet.contains( stack.pop().toString() ) );
    assertTrue( expectedSet.contains( stack.pop().toString() ) );
  }
  
  public void testFieldTermStackIndex1w2wSearch1term() throws Exception {
    makeIndex1w2w();
    
    FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 1, stack.termList.size() );
    assertEquals( "pc(3,5,1)", stack.pop().toString() );
  }
  
  public void testFieldTermStackIndex1w2wSearch1phrase() throws Exception {
    makeIndex1w2w();
    
    FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 2, stack.termList.size() );
    assertEquals( "personal(3,5,1)", stack.pop().toString() );
    assertEquals( "computer(3,5,2)", stack.pop().toString() );
  }
  
  public void testFieldTermStackIndex1w2wSearch1partial() throws Exception {
    makeIndex1w2w();
    
    FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 1, stack.termList.size() );
    assertEquals( "computer(3,5,2)", stack.pop().toString() );
  }
  
  public void testFieldTermStackIndex1w2wSearch1term1phrase() throws Exception {
    makeIndex1w2w();

    BooleanQuery bq = new BooleanQuery();
    bq.add( tq( "pc" ), Occur.SHOULD );
    bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
    FieldQuery fq = new FieldQuery( bq, true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 3, stack.termList.size() );
    Set<String> expectedSet = new HashSet();
    expectedSet.add( "pc(3,5,1)" );
    expectedSet.add( "personal(3,5,1)" );
    assertTrue( expectedSet.contains( stack.pop().toString() ) );
    assertTrue( expectedSet.contains( stack.pop().toString() ) );
    assertEquals( "computer(3,5,2)", stack.pop().toString() );
  }
  
  public void testFieldTermStackIndex2w1wSearch1term() throws Exception {
    makeIndex2w1w();
    
    FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 1, stack.termList.size() );
    assertEquals( "pc(3,20,1)", stack.pop().toString() );
  }
  
  public void testFieldTermStackIndex2w1wSearch1phrase() throws Exception {
    makeIndex2w1w();
    
    FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 2, stack.termList.size() );
    assertEquals( "personal(3,20,1)", stack.pop().toString() );
    assertEquals( "computer(3,20,2)", stack.pop().toString() );
  }
  
  public void testFieldTermStackIndex2w1wSearch1partial() throws Exception {
    makeIndex2w1w();
    
    FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 1, stack.termList.size() );
    assertEquals( "computer(3,20,2)", stack.pop().toString() );
  }
  
  public void testFieldTermStackIndex2w1wSearch1term1phrase() throws Exception {
    makeIndex2w1w();

    BooleanQuery bq = new BooleanQuery();
    bq.add( tq( "pc" ), Occur.SHOULD );
    bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
    FieldQuery fq = new FieldQuery( bq, true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    assertEquals( 3, stack.termList.size() );
    Set<String> expectedSet = new HashSet();
    expectedSet.add( "pc(3,20,1)" );
    expectedSet.add( "personal(3,20,1)" );
    assertTrue( expectedSet.contains( stack.pop().toString() ) );
    assertTrue( expectedSet.contains( stack.pop().toString() ) );
    assertEquals( "computer(3,20,2)", stack.pop().toString() );
  }
  
  public void testFieldPhraseListIndex1w2wSearch1phrase() throws Exception {
    makeIndex1w2w();
    
    FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    FieldPhraseList fpl = new FieldPhraseList( stack, fq );
    assertEquals( 1, fpl.phraseList.size() );
    assertEquals( "personalcomputer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() );
    assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
    assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );
  }
  
  public void testFieldPhraseListIndex1w2wSearch1partial() throws Exception {
    makeIndex1w2w();
    
    FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    FieldPhraseList fpl = new FieldPhraseList( stack, fq );
    assertEquals( 1, fpl.phraseList.size() );
    assertEquals( "computer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() );
    assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
    assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );
  }
  
  public void testFieldPhraseListIndex1w2wSearch1term1phrase() throws Exception {
    makeIndex1w2w();

    BooleanQuery bq = new BooleanQuery();
    bq.add( tq( "pc" ), Occur.SHOULD );
    bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
    FieldQuery fq = new FieldQuery( bq, true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    FieldPhraseList fpl = new FieldPhraseList( stack, fq );
    assertEquals( 1, fpl.phraseList.size() );
    assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,5))" ) > 0 );
    assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
    assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );
  }
  
  public void testFieldPhraseListIndex2w1wSearch1term() throws Exception {
    makeIndex2w1w();
    
    FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    FieldPhraseList fpl = new FieldPhraseList( stack, fq );
    assertEquals( 1, fpl.phraseList.size() );
    assertEquals( "pc(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );
    assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
    assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
  }
  
  public void testFieldPhraseListIndex2w1wSearch1phrase() throws Exception {
    makeIndex2w1w();
    
    FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    FieldPhraseList fpl = new FieldPhraseList( stack, fq );
    assertEquals( 1, fpl.phraseList.size() );
    assertEquals( "personalcomputer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );
    assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
    assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
  }
  
  public void testFieldPhraseListIndex2w1wSearch1partial() throws Exception {
    makeIndex2w1w();
    
    FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    FieldPhraseList fpl = new FieldPhraseList( stack, fq );
    assertEquals( 1, fpl.phraseList.size() );
    assertEquals( "computer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );
    assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
    assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
  }
  
  public void testFieldPhraseListIndex2w1wSearch1term1phrase() throws Exception {
    makeIndex2w1w();

    BooleanQuery bq = new BooleanQuery();
    bq.add( tq( "pc" ), Occur.SHOULD );
    bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
    FieldQuery fq = new FieldQuery( bq, true, true );
    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
    FieldPhraseList fpl = new FieldPhraseList( stack, fq );
    assertEquals( 1, fpl.phraseList.size() );
    assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,20))" ) > 0 );
    assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
    assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
  }

  private void makeIndex1w() throws Exception {
    //           11111111112
    // 012345678901234567890
    // I'll buy a Macintosh
    //            Mac
    //            MacBook
    // 0    1   2 3
    makeSynonymIndex( "I'll buy a Macintosh",
        t("I'll",0,4),
        t("buy",5,8),
        t("a",9,10),
        t("Macintosh",11,20),t("Mac",11,20,0),t("MacBook",11,20,0));
  }

  private void makeIndex1w2w() throws Exception {
    //           1111111
    // 01234567890123456
    // My pc was broken
    //    personal computer
    // 0  1  2   3
    makeSynonymIndex( "My pc was broken",
        t("My",0,2),
        t("pc",3,5),t("personal",3,5,0),t("computer",3,5),
        t("was",6,9),
        t("broken",10,16));
  }

  private void makeIndex2w1w() throws Exception {
    //           1111111111222222222233
    // 01234567890123456789012345678901
    // My personal computer was broken
    //    pc
    // 0  1        2        3   4
    makeSynonymIndex( "My personal computer was broken",
        t("My",0,2),
        t("personal",3,20),t("pc",3,20,0),t("computer",3,20),
        t("was",21,24),
        t("broken",25,31));
  }
  
  void makeSynonymIndex( String value, Token... tokens ) throws Exception {
    Analyzer analyzer = new TokenArrayAnalyzer( tokens );
    make1dmfIndex( analyzer, value );
  }

  public static Token t( String text, int startOffset, int endOffset ){
    return t( text, startOffset, endOffset, 1 );
  }
  
  public static Token t( String text, int startOffset, int endOffset, int positionIncrement ){
    Token token = new Token( text, startOffset, endOffset );
    token.setPositionIncrement( positionIncrement );
    return token;
  }
  
  public static final class TokenArrayAnalyzer extends Analyzer {
    Token[] tokens;
    public TokenArrayAnalyzer( Token... tokens ){
      this.tokens = tokens;
    }
    
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {      
      TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
        final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
        int p = 0;
        
        @Override
        public boolean incrementToken() throws IOException {
          if( p >= tokens.length ) return false;
          clearAttributes();
          tokens[p++].copyTo(reusableToken);
          return true;
        }
      };
      return ts;
    }
  }
}