Lucene example source code file (HighFreqTerms.java)

This example Lucene source code file (HighFreqTerms.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Java - Lucene tags/keywords

comparator, defaultnumterms, exception, exception, includetermfreqs, io, string, term, term, termenum, terminfowitfqueue, terminfowitfqueue, termstats, termstats, totaltermfreqcomparatorsortdescending, util

The Lucene HighFreqTerms.java source code

package org.apache.lucene.misc;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.PriorityQueue;
import java.util.Arrays;
import java.util.Comparator;

import java.io.File;

/**
 * <code>HighFreqTerms class extracts the top n most frequent terms
 * (by document frequency ) from an existing Lucene index and reports their
 * document frequency.  If used with the -t flag it also reports their 
 * total tf (total number of occurences) in order of highest total tf
 */
public class HighFreqTerms {
  
   // The top numTerms will be displayed
  public static final int DEFAULTnumTerms = 100;
  public static int numTerms = DEFAULTnumTerms;
  
  public static void main(String[] args) throws Exception {
    IndexReader reader = null;
    FSDirectory dir = null;
    String field = null;
    boolean IncludeTermFreqs = false; 
   
    if (args.length == 0 || args.length > 4) {
      usage();
      System.exit(1);
    }     

    if (args.length > 0) {
      dir = FSDirectory.open(new File(args[0]));
    }
   
    for (int i = 1; i < args.length; i++) {
      if (args[i].equals("-t")) {
        IncludeTermFreqs = true;
      }
      else{
        try {
          numTerms = Integer.parseInt(args[i]);
        } catch (NumberFormatException e) {
          field=args[i];
        }
      }
    }
    
    
    reader = IndexReader.open(dir, true);
    TermStats[] terms = getHighFreqTerms(reader, numTerms, field);
    /*
     * Insert logic so it will only lookup totaltf if right arg
     * also change names as in flex
     */
    if (!IncludeTermFreqs) {
      //default HighFreqTerms behavior
      for (int i = 0; i < terms.length; i++) {
        System.out.printf("%s %,d \n",
            terms[i].term, terms[i].docFreq);
      }
    } else {

      TermStats[] termsWithTF = sortByTotalTermFreq(reader, terms);
      for (int i = 0; i < termsWithTF.length; i++) {
        System.out.printf("%s \t total_tf = %,d \t doc freq = %,d \n",
                          termsWithTF[i].term, termsWithTF[i].totalTermFreq, termsWithTF[i].docFreq);
      }
    }

    reader.close();
 }
  
  private static void usage() {
    System.out
        .println("\n\n"
            + "java org.apache.lucene.misc.HighFreqTerms <index dir> [-t] [number_terms] [field]\n\t -t: include totalTermFreq\n\n");
  } 

  /**
   * 
   * @param reader
   * @param numTerms
   * @param field
   * @return TermStats[] ordered by terms with highest docFreq first.
   * @throws Exception
   */
  public static TermStats[] getHighFreqTerms(IndexReader reader,
      int numTerms, String field) throws Exception {

    TermInfoWiTFQueue tiq = new TermInfoWiTFQueue(numTerms);
    if (field != null) {
      TermEnum terms = reader.terms(new Term(field));
      if (terms != null && terms.term() != null) {
        do {
          if (!terms.term().field().equals(field)) {
            break;
          }
          tiq.insertWithOverflow(new TermStats(terms.term(), terms.docFreq()));
        } while (terms.next());
      } else {
        System.out.println("No terms for field \"" + field + "\"");
      }
    } else {
      TermEnum terms = reader.terms();
      while (terms.next()) {
        tiq.insertWithOverflow(new TermStats(terms.term(), terms.docFreq()));
      }
    }

    TermStats[] result = new TermStats[tiq.size()];

    // we want highest first so we read the queue and populate the array
    // starting at the end and work backwards
    int count = tiq.size() - 1;
    while (tiq.size() != 0) {
      result[count] = tiq.pop();
      count--;
    }
    return result;
  }

  /**
   * Takes array of TermStats. For each term looks up the tf for each doc
   * containing the term and stores the total in the output array of TermStats.
   * Output array is sorted by highest total tf.
   * 
   * @param reader
   * @param terms
   *          TermStats[]
   * @return TermStats[]
   * @throws Exception
   */
 
  public static TermStats[] sortByTotalTermFreq(IndexReader reader, TermStats[] terms) throws Exception {
    TermStats[] ts = new TermStats[terms.length]; // array for sorting
    long totalTF;
    for (int i = 0; i < terms.length; i++) {
      totalTF = getTotalTermFreq(reader, terms[i].term);
      ts[i] = new TermStats( terms[i].term, terms[i].docFreq, totalTF);
    }
    
    Comparator<TermStats> c = new TotalTermFreqComparatorSortDescending();
    Arrays.sort(ts, c);
    
    return ts;
  }
  
  public static long getTotalTermFreq(IndexReader reader, Term term) throws Exception {
    long totalTF = 0;
    TermDocs td = reader.termDocs(term);
    while (td.next()) {
      totalTF += td.freq();
    }
    return totalTF;
  }
}


final class TermStats {
  public Term term;
  public int docFreq;
  public long totalTermFreq;
  
  public TermStats(Term t, int df) {
    this.term = t;
    this.docFreq = df;
  }
  
  public TermStats(Term t, int df, long tf) {
    this.term = t;
    this.docFreq = df;
    this.totalTermFreq = tf;
  }
}


/**
 * Priority queue for TermStats objects ordered by TermStats.docFreq
 **/
final class TermInfoWiTFQueue extends PriorityQueue<TermStats> {
  TermInfoWiTFQueue(int size) {
    initialize(size);
  }
  
  @Override
  protected boolean lessThan(TermStats termInfoA,
      TermStats termInfoB) {
    return termInfoA.docFreq < termInfoB.docFreq;
  }
}

/**
 * Comparator
 * 
 * Reverse of normal Comparator. i.e. returns 1 if a.totalTermFreq is less than
 * b.totalTermFreq So we can sort in descending order of totalTermFreq
 */
final class TotalTermFreqComparatorSortDescending implements Comparator<TermStats> {
  
  public int compare(TermStats a, TermStats b) {
    if (a.totalTermFreq < b.totalTermFreq) {
      return 1;
    } else if (a.totalTermFreq > b.totalTermFreq) {
      return -1;
    } else {
      return 0;
    }
  }
}