Lucene example source code file (WordlistLoader.java)

This example Lucene source code file (WordlistLoader.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Java - Lucene tags/keywords

bufferedreader, bufferedreader, filereader, hashmap, hashset, hashset, inputstreamreader, io, ioexception, ioexception, reader, set, string, string, utf-8, util

The Lucene WordlistLoader.java source code

package org.apache.lucene.analysis;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;

/**
 * Loader for text files that represent a list of stopwords.
 */
public class WordlistLoader {
 
  /**
   * Loads a text file associated with a given class (See
   * {@link Class#getResourceAsStream(String)}) and adds every line as an entry
   * to a {@link Set} (omitting leading and trailing whitespace). Every line of
   * the file should contain only one word. The words need to be in lower-case if
   * you make use of an Analyzer which uses LowerCaseFilter (like
   * StandardAnalyzer).
   * 
   * @param aClass
   *          a class that is associated with the given stopwordResource
   * @param stopwordResource
   *          name of the resource file associated with the given class
   * @return a {@link Set} with the file's words
   */
  public static Set<String> getWordSet(Class aClass, String stopwordResource)
      throws IOException {
    final Reader reader = new BufferedReader(new InputStreamReader(aClass
        .getResourceAsStream(stopwordResource), "UTF-8"));
    try {
      return getWordSet(reader);
    } finally {
      reader.close();
    }
  }
  
  /**
   * Loads a text file associated with a given class (See
   * {@link Class#getResourceAsStream(String)}) and adds every line as an entry
   * to a {@link Set} (omitting leading and trailing whitespace). Every line of
   * the file should contain only one word. The words need to be in lower-case if
   * you make use of an Analyzer which uses LowerCaseFilter (like
   * StandardAnalyzer).
   * 
   * @param aClass
   *          a class that is associated with the given stopwordResource
   * @param stopwordResource
   *          name of the resource file associated with the given class
   * @param comment
   *          the comment string to ignore
   * @return a {@link Set} with the file's words
   */
  public static Set<String> getWordSet(Class aClass,
      String stopwordResource, String comment) throws IOException {
    final Reader reader = new BufferedReader(new InputStreamReader(aClass
        .getResourceAsStream(stopwordResource), "UTF-8"));
    try {
      return getWordSet(reader, comment);
    } finally {
      reader.close();
    }
  }
  
  /**
   * Loads a text file and adds every line as an entry to a HashSet (omitting
   * leading and trailing whitespace). Every line of the file should contain only
   * one word. The words need to be in lowercase if you make use of an
   * Analyzer which uses LowerCaseFilter (like StandardAnalyzer).
   *
   * @param wordfile File containing the wordlist
   * @return A HashSet with the file's words
   */
  public static HashSet<String> getWordSet(File wordfile) throws IOException {
    FileReader reader = null;
    try {
      reader = new FileReader(wordfile);
      return getWordSet(reader);
    }
    finally {
      if (reader != null)
        reader.close();
    }
  }

  /**
   * Loads a text file and adds every non-comment line as an entry to a HashSet (omitting
   * leading and trailing whitespace). Every line of the file should contain only
   * one word. The words need to be in lowercase if you make use of an
   * Analyzer which uses LowerCaseFilter (like StandardAnalyzer).
   *
   * @param wordfile File containing the wordlist
   * @param comment The comment string to ignore
   * @return A HashSet with the file's words
   */
  public static HashSet<String> getWordSet(File wordfile, String comment) throws IOException {
    FileReader reader = null;
    try {
      reader = new FileReader(wordfile);
      return getWordSet(reader, comment);
    }
    finally {
      if (reader != null)
        reader.close();
    }
  }


  /**
   * Reads lines from a Reader and adds every line as an entry to a HashSet (omitting
   * leading and trailing whitespace). Every line of the Reader should contain only
   * one word. The words need to be in lowercase if you make use of an
   * Analyzer which uses LowerCaseFilter (like StandardAnalyzer).
   *
   * @param reader Reader containing the wordlist
   * @return A HashSet with the reader's words
   */
  public static HashSet<String> getWordSet(Reader reader) throws IOException {
    final HashSet<String> result = new HashSet();
    BufferedReader br = null;
    try {
      if (reader instanceof BufferedReader) {
        br = (BufferedReader) reader;
      } else {
        br = new BufferedReader(reader);
      }
      String word = null;
      while ((word = br.readLine()) != null) {
        result.add(word.trim());
      }
    }
    finally {
      if (br != null)
        br.close();
    }
    return result;
  }

  /**
   * Reads lines from a Reader and adds every non-comment line as an entry to a HashSet (omitting
   * leading and trailing whitespace). Every line of the Reader should contain only
   * one word. The words need to be in lowercase if you make use of an
   * Analyzer which uses LowerCaseFilter (like StandardAnalyzer).
   *
   * @param reader Reader containing the wordlist
   * @param comment The string representing a comment.
   * @return A HashSet with the reader's words
   */
  public static HashSet<String> getWordSet(Reader reader, String comment) throws IOException {
    final HashSet<String> result = new HashSet();
    BufferedReader br = null;
    try {
      if (reader instanceof BufferedReader) {
        br = (BufferedReader) reader;
      } else {
        br = new BufferedReader(reader);
      }
      String word = null;
      while ((word = br.readLine()) != null) {
        if (word.startsWith(comment) == false){
          result.add(word.trim());
        }
      }
    }
    finally {
      if (br != null)
        br.close();
    }
    return result;
  }

  /**
   * Loads a text file in Snowball format associated with a given class (See
   * {@link Class#getResourceAsStream(String)}) and adds all words as entries to
   * a {@link Set}. The words need to be in lower-case if you make use of an
   * Analyzer which uses LowerCaseFilter (like StandardAnalyzer).
   * 
   * @param aClass a class that is associated with the given stopwordResource
   * @param stopwordResource name of the resource file associated with the given
   *          class
   * @return a {@link Set} with the file's words
   * @see #getSnowballWordSet(Reader)
   */
  public static Set<String> getSnowballWordSet(Class aClass,
      String stopwordResource) throws IOException {
    final Reader reader = new BufferedReader(new InputStreamReader(aClass
        .getResourceAsStream(stopwordResource), "UTF-8"));
    try {
      return getSnowballWordSet(reader);
    } finally {
      reader.close();
    }
  }
  
  /**
   * Reads stopwords from a stopword list in Snowball format.
   * <p>
   * The snowball format is the following:
   * <ul>
   * <li>Lines may contain multiple words separated by whitespace.
   * <li>The comment character is the vertical line (|).
   * <li>Lines may contain trailing comments.
   * </ul>
   * </p>
   * 
   * @param reader Reader containing a Snowball stopword list
   * @return A Set with the reader's words
   */
  public static Set<String> getSnowballWordSet(Reader reader)
      throws IOException {
    final Set<String> result = new HashSet();
    BufferedReader br = null;
    try {
      if (reader instanceof BufferedReader) {
        br = (BufferedReader) reader;
      } else {
        br = new BufferedReader(reader);
      }
      String line = null;
      while ((line = br.readLine()) != null) {
        int comment = line.indexOf('|');
        if (comment >= 0) line = line.substring(0, comment);
        String words[] = line.split("\\s+");
        for (int i = 0; i < words.length; i++)
          if (words[i].length() > 0) result.add(words[i]);
      }
    } finally {
      if (br != null) br.close();
    }
    return result;
  }


  /**
   * Reads a stem dictionary. Each line contains:
   * <pre>word\tstem

* (i.e. two tab separated words) * * @return stem dictionary that overrules the stemming algorithm * @throws IOException */ public static HashMap<String, String> getStemDict(File wordstemfile) throws IOException { if (wordstemfile == null) throw new NullPointerException("wordstemfile may not be null"); final HashMap<String, String> result = new HashMap(); BufferedReader br = null; try { br = new BufferedReader(new FileReader(wordstemfile)); String line; while ((line = br.readLine()) != null) { String[] wordstem = line.split("\t", 2); result.put(wordstem[0], wordstem[1]); } } finally { if(br != null) br.close(); } return result; } }

Other Lucene examples (source code examples)

Here is a short list of links related to this Lucene WordlistLoader.java source code file:

•  The search page
•  Other Lucene source code examples at this package level
•  Click here to learn more about this project