Lucene example source code file (TestHighFreqTerms.java)

This example Lucene source code file (TestHighFreqTerms.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Java - Lucene tags/keywords

beforeclass, document, exception, exception, field_1, field_1, indexreader, indexwriter, mockanalyzer, string, string, term, termstats, termstats

The Lucene TestHighFreqTerms.java source code

package org.apache.lucene.misc;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.junit.AfterClass;
import org.junit.BeforeClass;

public class TestHighFreqTerms extends LuceneTestCase {
 
  private static IndexWriter writer =null;
  private static Directory dir = null;
  private static IndexReader reader =null;
  
  @BeforeClass
  public static void setUpClass() throws Exception {
    dir = newDirectory();
    writer = new IndexWriter(dir, newIndexWriterConfig(random,
       TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false))
       .setMaxBufferedDocs(2));
   indexDocs(writer);
   reader = IndexReader.open(dir, true);
  }
  
  @AfterClass
  public static void tearDownClass() throws Exception{
    reader.close();
    dir.close();
    dir = null;
    reader = null;
  }

/******************** Tests for getHighFreqTerms **********************************/
  
  // test without specifying field (i.e. if we pass in field=null it should examine all fields)
  // the term "diff" in the field "different_field" occurs 20 times and is the highest df term
  public void testFirstTermHighestDocFreqAllFields () throws Exception{
    int numTerms = 12;
    String field =null;
    TermStats[] terms = HighFreqTerms.getHighFreqTerms(reader, numTerms, field);
    assertEquals("Term with highest docfreq is first", 20,terms[0].docFreq );
  }
  
  public void testFirstTermHighestDocFreq () throws Exception{
    int numTerms = 12;
    String field="FIELD_1";
    TermStats[] terms = HighFreqTerms.getHighFreqTerms(reader, numTerms, field);
    assertEquals("Term with highest docfreq is first", 10,terms[0].docFreq );
  }
  public void testOrderedByDocFreqDescending () throws Exception{
    int numTerms = 12;
    String field="FIELD_1";
    TermStats[] terms = HighFreqTerms.getHighFreqTerms(reader, numTerms, field);
    for (int i = 0; i < terms.length; i++) {
      if (i >0){
       assertTrue ("out of order " + terms[i-1].docFreq + "should be >= " + terms[i].docFreq,terms[i-1].docFreq >= terms[i].docFreq);
      }
    }    
  }
  
  public void testNumTerms () throws Exception{
    int numTerms = 12;
    String field = null;
    TermStats[] terms = HighFreqTerms.getHighFreqTerms(reader, numTerms, field);
    assertEquals("length of terms array equals numTerms :" + numTerms, numTerms, terms.length);
  }
    
  public void testGetHighFreqTerms () throws Exception{
    int numTerms=12;
    String field="FIELD_1";
    TermStats[] terms = HighFreqTerms.getHighFreqTerms(reader, numTerms, field);
       
    for (int i = 0; i < terms.length; i++) {
      String termtext = terms[i].term.text();
      // hardcoded highTF or highTFmedDF
      if (termtext.contains("highTF")) {
        if (termtext.contains("medDF")) {
          assertEquals("doc freq is not as expected", 5, terms[i].docFreq);
        } else {
          assertEquals("doc freq is not as expected", 1, terms[i].docFreq);
        }
      } else {
        int n = Integer.parseInt(termtext);
        assertEquals("doc freq is not as expected", getExpecteddocFreq(n),
            terms[i].docFreq);
      }
    }
  }
  
  /********************Test sortByTotalTermFreq**********************************/
  
  public void testFirstTermHighestTotalTermFreq () throws Exception{
    int numTerms = 20;
    String field = null;
    TermStats[] terms = HighFreqTerms.getHighFreqTerms(reader, numTerms, field);
    TermStats[] termsWithTotalTermFreq = HighFreqTerms.sortByTotalTermFreq(reader, terms);
    assertEquals("Term with highest totalTermFreq is first",200, termsWithTotalTermFreq[0].totalTermFreq);
  }
  public void testFirstTermHighestTotalTermFreqDifferentField () throws Exception{
    int numTerms = 20;
    String field = "different_field";
    TermStats[] terms = HighFreqTerms.getHighFreqTerms(reader, numTerms, field);
    TermStats[] termsWithTotalTermFreq = HighFreqTerms.sortByTotalTermFreq(reader, terms);
    assertEquals("Term with highest totalTermFreq is first"+ termsWithTotalTermFreq[0].term.text(),150, termsWithTotalTermFreq[0].totalTermFreq);
  }
  
  public void testOrderedByTermFreqDescending () throws Exception{
    int numTerms = 12;
    String field = "FIELD_1";
    TermStats[] terms = HighFreqTerms.getHighFreqTerms(reader, numTerms, field);
    TermStats[] termsWithTF = HighFreqTerms.sortByTotalTermFreq(reader, terms);
 
  for (int i = 0; i < termsWithTF.length; i++) {
    // check that they are sorted by descending termfreq order
    if (i >0){
      assertTrue ("out of order" +termsWithTF[i-1]+ " > " +termsWithTF[i],termsWithTF[i-1].totalTermFreq > termsWithTF[i].totalTermFreq);
     }
    } 
  }
  
  public void testGetTermFreqOrdered () throws Exception{
    int numTerms = 12;
    String field = "FIELD_1";
    TermStats[] terms = HighFreqTerms.getHighFreqTerms(reader, numTerms, field);
    TermStats[] termsWithTF = HighFreqTerms.sortByTotalTermFreq(reader, terms);
   
    for (int i = 0; i < termsWithTF.length; i++) {
      String text = termsWithTF[i].term.text();
      if (text.contains("highTF")) {
        if (text.contains("medDF")) {
          assertEquals("total term freq is expected", 125,
              termsWithTF[i].totalTermFreq);
        } else {
          assertEquals("total term freq is expected", 200,
              termsWithTF[i].totalTermFreq);
        }
        
      } else {
        int n = Integer.parseInt(text);
        assertEquals("doc freq is expected", getExpecteddocFreq(n),
            termsWithTF[i].docFreq);
        assertEquals("total term freq is expected", getExpectedtotalTermFreq(n),
            termsWithTF[i].totalTermFreq);
      }
    }
  }
    
    /********************Tests for getTotalTermFreq**********************************/
    
  public void testGetTotalTermFreq() throws Exception{
    String termtext ="highTF";
    String field = "FIELD_1";
    Term term = new Term(field,termtext);
    long totalTermFreq = HighFreqTerms.getTotalTermFreq(reader, term);
    assertEquals("highTf tf should be 200",200,totalTermFreq);
  }
    
  public void testGetTotalTermFreqBadTerm() throws Exception{
    String termtext ="foobar";
    String field = "FIELD_1";
    Term term = new Term(field,termtext);
    long totalTermFreq = HighFreqTerms.getTotalTermFreq(reader, term);
    assertEquals("totalTermFreq should be 0 for term not in index",0,totalTermFreq);
  }
    /********************Testing Utils**********************************/
    
  private static void indexDocs(IndexWriter writer) throws Exception {

     /**
      * Generate 10 documents where term n  has a docFreq of n and a totalTermFreq of n*2 (squared). 
      */
    for (int i = 1; i <= 10; i++) {
      Document doc = new Document();
      String content = getContent(i);
    
      doc.add(newField(random, "FIELD_1", content, Field.Store.YES,Field.Index.ANALYZED, Field.TermVector.NO));
      //add a different field
      doc.add(newField(random, "different_field", "diff", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
      writer.addDocument(doc);
    }
    
    //add 10 more docs with the term "diff" this will make it have the highest docFreq if we don't ask for the
    //highest freq terms for a specific field.
    for (int i = 1; i <= 10; i++) {
      Document doc = new Document();
      doc.add(newField(random, "different_field", "diff", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
      writer.addDocument(doc);
    }
    // add some docs where tf < df so we can see if sorting works
    // highTF low df
    int highTF = 200;
    Document doc = new Document();
    String content = "";
    for (int i = 0; i < highTF; i++) {
      content += "highTF ";
    }
    doc.add(newField(random, "FIELD_1", content, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
    writer.addDocument(doc);
    // highTF medium df =5
    int medium_df = 5;
    for (int i = 0; i < medium_df; i++) {
      int tf = 25;
      Document newdoc = new Document();
      String newcontent = "";
      for (int j = 0; j < tf; j++) {
        newcontent += "highTFmedDF ";
      }
      newdoc.add(newField(random, "FIELD_1", newcontent, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
      writer.addDocument(newdoc);
    }
    // add a doc with high tf in field different_field
    int targetTF =150;
    doc = new Document();
    content = "";
    for (int i = 0; i < targetTF; i++) {
      content += "TF150 ";
    }
    doc.add(newField(random, "different_field", content, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
    writer.addDocument(doc);
    writer.close();
    
  }
  
/**
 *  getContent
 *  return string containing numbers 1 to i with each number n occurring n times.
 *  i.e. for input of 3 return string "3 3 3 2 2 1" 
 */
    
  private static String getContent(int i) {
    String s = "";
    for (int j = 10; j >= i; j--) {
      for (int k = 0; k < j; k++) {
        // if j is 3 we return "3 3 3"
        s += String.valueOf(j) + " ";
      }
    }
    return s;
  }
  
  private static int getExpectedtotalTermFreq(int i) {
    return getExpecteddocFreq(i) * i;
  }
  
  private static int getExpecteddocFreq(int i) {
    return i;
  }
}