Lucene example source code file (ICUTransformFilter.java)

This example Lucene source code file (ICUTransformFilter.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Java - Lucene tags/keywords

chartermattribute, chartermattribute, icutransformfilter, icutransformfilter, io, ioexception, override, replaceable, replaceabletermattribute, replaceabletermattribute, string, tokenfilter, transliterator, transliterator, unicodeset

The Lucene ICUTransformFilter.java source code

package org.apache.lucene.analysis.icu;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;

import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

import com.ibm.icu.text.Replaceable;
import com.ibm.icu.text.Transliterator;
import com.ibm.icu.text.UTF16;
import com.ibm.icu.text.UnicodeSet;

/**
 * A {@link TokenFilter} that transforms text with ICU.
 * <p>
 * ICU provides text-transformation functionality via its Transliteration API.
 * Although script conversion is its most common use, a Transliterator can
 * actually perform a more general class of tasks. In fact, Transliterator
 * defines a very general API which specifies only that a segment of the input
 * text is replaced by new text. The particulars of this conversion are
 * determined entirely by subclasses of Transliterator.
 * </p>
 * <p>
 * Some useful transformations for search are built-in:
 * <ul>
 * <li>Conversion from Traditional to Simplified Chinese characters
 * <li>Conversion from Hiragana to Katakana
 * <li>Conversion from Fullwidth to Halfwidth forms.
 * <li>Script conversions, for example Serbian Cyrillic to Latin
 * </ul>
 * </p>
 * <p>
 * Example usage: <blockquote>stream = new ICUTransformFilter(stream,
 * Transliterator.getInstance("Traditional-Simplified"));</blockquote>
 * </p>
 * For more details, see the <a
 * href="http://userguide.icu-project.org/transforms/general">ICU User
 * Guide</a>.
 */
public final class ICUTransformFilter extends TokenFilter {
  // Transliterator to transform the text
  private final Transliterator transform;

  // Reusable position object
  private final Transliterator.Position position = new Transliterator.Position();

  // term attribute, will be updated with transformed text.
  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);

  // Wraps a termAttribute around the replaceable interface.
  private final ReplaceableTermAttribute replaceableAttribute = new ReplaceableTermAttribute();

  /**
   * Create a new ICUTransformFilter that transforms text on the given stream.
   * 
   * @param input {@link TokenStream} to filter.
   * @param transform Transliterator to transform the text.
   */
  public ICUTransformFilter(TokenStream input, Transliterator transform) {
    super(input);
    this.transform = transform;

    /* 
     * This is cheating, but speeds things up a lot.
     * If we wanted to use pkg-private APIs we could probably do better.
     */
    if (transform.getFilter() == null && transform instanceof com.ibm.icu.text.RuleBasedTransliterator) {
      final UnicodeSet sourceSet = transform.getSourceSet();
      if (sourceSet != null && !sourceSet.isEmpty())
        transform.setFilter(sourceSet);
    }
  }

  @Override
  public boolean incrementToken() throws IOException {
    /*
     * Wrap around replaceable. clear the positions, and transliterate.
     */
    if (input.incrementToken()) {
      replaceableAttribute.setText(termAtt);
      
      final int length = termAtt.length(); 
      position.start = 0;
      position.limit = length;
      position.contextStart = 0;
      position.contextLimit = length;

      transform.filteredTransliterate(replaceableAttribute, position, false);
      return true;
    } else {
      return false;
    }
  }
  
  /**
   * Wrap a {@link CharTermAttribute} with the Replaceable API.
   */
  final class ReplaceableTermAttribute implements Replaceable {
    private char buffer[];
    private int length;
    private CharTermAttribute token;

    void setText(final CharTermAttribute token) {
      this.token = token;
      this.buffer = token.buffer();
      this.length = token.length();
    }

    public int char32At(int pos) {
      return UTF16.charAt(buffer, 0, length, pos);
    }

    public char charAt(int pos) {
      return buffer[pos];
    }

    public void copy(int start, int limit, int dest) {
      char text[] = new char[limit - start];
      getChars(start, limit, text, 0);
      replace(dest, dest, text, 0, limit - start);
    }

    public void getChars(int srcStart, int srcLimit, char[] dst, int dstStart) {
      System.arraycopy(buffer, srcStart, dst, dstStart, srcLimit - srcStart);
    }

    public boolean hasMetaData() {
      return false;
    }

    public int length() {
      return length;
    }

    public void replace(int start, int limit, String text) {
      final int charsLen = text.length();
      final int newLength = shiftForReplace(start, limit, charsLen);
      // insert the replacement text
      text.getChars(0, charsLen, buffer, start);
      token.setLength(length = newLength);
    }

    public void replace(int start, int limit, char[] text, int charsStart,
        int charsLen) {
      // shift text if necessary for the replacement
      final int newLength = shiftForReplace(start, limit, charsLen);
      // insert the replacement text
      System.arraycopy(text, charsStart, buffer, start, charsLen);
      token.setLength(length = newLength);
    }

    /** shift text (if necessary) for a replacement operation */
    private int shiftForReplace(int start, int limit, int charsLen) {
      final int replacementLength = limit - start;
      final int newLength = length - replacementLength + charsLen;
      // resize if necessary
      if (newLength > length)
        buffer = token.resizeBuffer(newLength);
      // if the substring being replaced is longer or shorter than the
      // replacement, need to shift things around
      if (replacementLength != charsLen && limit < length)
        System.arraycopy(buffer, limit, buffer, start + charsLen, length - limit);
      return newLength;
    }
  }
}