Lucene example source code file (HyphenationTree.java)

This example Lucene source code file (HyphenationTree.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Java - Lucene tags/keywords

arraylist, bytevector, hashmap, hyphenation, hyphenation, hyphenationexception, hyphenationtree, io, net, network, patternparser, sax, string, string, stringbuilder, stringbuilder, ternarytree, ternarytree, util

The Lucene HyphenationTree.java source code

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.analysis.compound.hyphenation;

import java.io.File;
import java.io.Serializable;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;

import org.xml.sax.InputSource;

/**
 * This tree structure stores the hyphenation patterns in an efficient way for
 * fast lookup. It provides the provides the method to hyphenate a word.
 * 
 * This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified. 
 */
public class HyphenationTree extends TernaryTree implements PatternConsumer,
    Serializable {

  private static final long serialVersionUID = -7842107987915665573L;

  /**
   * value space: stores the interletter values
   */
  protected ByteVector vspace;

  /**
   * This map stores hyphenation exceptions
   */
  protected HashMap<String,ArrayList stoplist;

  /**
   * This map stores the character classes
   */
  protected TernaryTree classmap;

  /**
   * Temporary map to store interletter values on pattern loading.
   */
  private transient TernaryTree ivalues;

  public HyphenationTree() {
    stoplist = new HashMap<String,ArrayList(23); // usually a small table
    classmap = new TernaryTree();
    vspace = new ByteVector();
    vspace.alloc(1); // this reserves index 0, which we don't use
  }

  /**
   * Packs the values by storing them in 4 bits, two values into a byte Values
   * range is from 0 to 9. We use zero as terminator, so we'll add 1 to the
   * value.
   * 
   * @param values a string of digits from '0' to '9' representing the
   *        interletter values.
   * @return the index into the vspace array where the packed values are stored.
   */
  protected int packValues(String values) {
    int i, n = values.length();
    int m = (n & 1) == 1 ? (n >> 1) + 2 : (n >> 1) + 1;
    int offset = vspace.alloc(m);
    byte[] va = vspace.getArray();
    for (i = 0; i < n; i++) {
      int j = i >> 1;
      byte v = (byte) ((values.charAt(i) - '0' + 1) & 0x0f);
      if ((i & 1) == 1) {
        va[j + offset] = (byte) (va[j + offset] | v);
      } else {
        va[j + offset] = (byte) (v << 4); // big endian
      }
    }
    va[m - 1 + offset] = 0; // terminator
    return offset;
  }

  protected String unpackValues(int k) {
    StringBuilder buf = new StringBuilder();
    byte v = vspace.get(k++);
    while (v != 0) {
      char c = (char) ((v >>> 4) - 1 + '0');
      buf.append(c);
      c = (char) (v & 0x0f);
      if (c == 0) {
        break;
      }
      c = (char) (c - 1 + '0');
      buf.append(c);
      v = vspace.get(k++);
    }
    return buf.toString();
  }

  /**
   * Read hyphenation patterns from an XML file.
   * 
   * @param f the filename
   * @throws HyphenationException In case the parsing fails
   */
  public void loadPatterns(File f) throws HyphenationException {
    try {
      InputSource src = new InputSource(f.toURL().toExternalForm());
      loadPatterns(src);
    } catch (MalformedURLException e) {
      throw new HyphenationException("Error converting the File '" + f
          + "' to a URL: " + e.getMessage());
    }
  }

  /**
   * Read hyphenation patterns from an XML file.
   * 
   * @param source the InputSource for the file
   * @throws HyphenationException In case the parsing fails
   */
  public void loadPatterns(InputSource source) throws HyphenationException {
    PatternParser pp = new PatternParser(this);
    ivalues = new TernaryTree();

    pp.parse(source);

    // patterns/values should be now in the tree
    // let's optimize a bit
    trimToSize();
    vspace.trimToSize();
    classmap.trimToSize();

    // get rid of the auxiliary map
    ivalues = null;
  }

  public String findPattern(String pat) {
    int k = super.find(pat);
    if (k >= 0) {
      return unpackValues(k);
    }
    return "";
  }

  /**
   * String compare, returns 0 if equal or t is a substring of s
   */
  protected int hstrcmp(char[] s, int si, char[] t, int ti) {
    for (; s[si] == t[ti]; si++, ti++) {
      if (s[si] == 0) {
        return 0;
      }
    }
    if (t[ti] == 0) {
      return 0;
    }
    return s[si] - t[ti];
  }

  protected byte[] getValues(int k) {
    StringBuilder buf = new StringBuilder();
    byte v = vspace.get(k++);
    while (v != 0) {
      char c = (char) ((v >>> 4) - 1);
      buf.append(c);
      c = (char) (v & 0x0f);
      if (c == 0) {
        break;
      }
      c = (char) (c - 1);
      buf.append(c);
      v = vspace.get(k++);
    }
    byte[] res = new byte[buf.length()];
    for (int i = 0; i < res.length; i++) {
      res[i] = (byte) buf.charAt(i);
    }
    return res;
  }

  /**
   * <p>
   * Search for all possible partial matches of word starting at index an update
   * interletter values. In other words, it does something like:
   * </p>
   * <code>
   * for(i=0; i<patterns.length; i++) {
   * if ( word.substring(index).startsWidth(patterns[i]) )
   * update_interletter_values(patterns[i]);
   * }
   * </code>
   * <p>
   * But it is done in an efficient way since the patterns are stored in a
   * ternary tree. In fact, this is the whole purpose of having the tree: doing
   * this search without having to test every single pattern. The number of
   * patterns for languages such as English range from 4000 to 10000. Thus,
   * doing thousands of string comparisons for each word to hyphenate would be
   * really slow without the tree. The tradeoff is memory, but using a ternary
   * tree instead of a trie, almost halves the the memory used by Lout or TeX.
   * It's also faster than using a hash table
   * </p>
   * 
   * @param word null terminated word to match
   * @param index start index from word
   * @param il interletter values array to update
   */
  protected void searchPatterns(char[] word, int index, byte[] il) {
    byte[] values;
    int i = index;
    char p, q;
    char sp = word[i];
    p = root;

    while (p > 0 && p < sc.length) {
      if (sc[p] == 0xFFFF) {
        if (hstrcmp(word, i, kv.getArray(), lo[p]) == 0) {
          values = getValues(eq[p]); // data pointer is in eq[]
          int j = index;
          for (int k = 0; k < values.length; k++) {
            if (j < il.length && values[k] > il[j]) {
              il[j] = values[k];
            }
            j++;
          }
        }
        return;
      }
      int d = sp - sc[p];
      if (d == 0) {
        if (sp == 0) {
          break;
        }
        sp = word[++i];
        p = eq[p];
        q = p;

        // look for a pattern ending at this position by searching for
        // the null char ( splitchar == 0 )
        while (q > 0 && q < sc.length) {
          if (sc[q] == 0xFFFF) { // stop at compressed branch
            break;
          }
          if (sc[q] == 0) {
            values = getValues(eq[q]);
            int j = index;
            for (int k = 0; k < values.length; k++) {
              if (j < il.length && values[k] > il[j]) {
                il[j] = values[k];
              }
              j++;
            }
            break;
          } else {
            q = lo[q];

            /**
             * actually the code should be: q = sc[q] < 0 ? hi[q] : lo[q]; but
             * java chars are unsigned
             */
          }
        }
      } else {
        p = d < 0 ? lo[p] : hi[p];
      }
    }
  }

  /**
   * Hyphenate word and return a Hyphenation object.
   * 
   * @param word the word to be hyphenated
   * @param remainCharCount Minimum number of characters allowed before the
   *        hyphenation point.
   * @param pushCharCount Minimum number of characters allowed after the
   *        hyphenation point.
   * @return a {@link Hyphenation Hyphenation} object representing the
   *         hyphenated word or null if word is not hyphenated.
   */
  public Hyphenation hyphenate(String word, int remainCharCount,
      int pushCharCount) {
    char[] w = word.toCharArray();
    return hyphenate(w, 0, w.length, remainCharCount, pushCharCount);
  }

  /**
   * w = "****nnllllllnnn*****", where n is a non-letter, l is a letter, all n
   * may be absent, the first n is at offset, the first l is at offset +
   * iIgnoreAtBeginning; word = ".llllll.'\0'***", where all l in w are copied
   * into word. In the first part of the routine len = w.length, in the second
   * part of the routine len = word.length. Three indices are used: index(w),
   * the index in w, index(word), the index in word, letterindex(word), the
   * index in the letter part of word. The following relations exist: index(w) =
   * offset + i - 1 index(word) = i - iIgnoreAtBeginning letterindex(word) =
   * index(word) - 1 (see first loop). It follows that: index(w) - index(word) =
   * offset - 1 + iIgnoreAtBeginning index(w) = letterindex(word) + offset +
   * iIgnoreAtBeginning
   */

  /**
   * Hyphenate word and return an array of hyphenation points.
   * 
   * @param w char array that contains the word
   * @param offset Offset to first character in word
   * @param len Length of word
   * @param remainCharCount Minimum number of characters allowed before the
   *        hyphenation point.
   * @param pushCharCount Minimum number of characters allowed after the
   *        hyphenation point.
   * @return a {@link Hyphenation Hyphenation} object representing the
   *         hyphenated word or null if word is not hyphenated.
   */
  public Hyphenation hyphenate(char[] w, int offset, int len,
      int remainCharCount, int pushCharCount) {
    int i;
    char[] word = new char[len + 3];

    // normalize word
    char[] c = new char[2];
    int iIgnoreAtBeginning = 0;
    int iLength = len;
    boolean bEndOfLetters = false;
    for (i = 1; i <= len; i++) {
      c[0] = w[offset + i - 1];
      int nc = classmap.find(c, 0);
      if (nc < 0) { // found a non-letter character ...
        if (i == (1 + iIgnoreAtBeginning)) {
          // ... before any letter character
          iIgnoreAtBeginning++;
        } else {
          // ... after a letter character
          bEndOfLetters = true;
        }
        iLength--;
      } else {
        if (!bEndOfLetters) {
          word[i - iIgnoreAtBeginning] = (char) nc;
        } else {
          return null;
        }
      }
    }
    len = iLength;
    if (len < (remainCharCount + pushCharCount)) {
      // word is too short to be hyphenated
      return null;
    }
    int[] result = new int[len + 1];
    int k = 0;

    // check exception list first
    String sw = new String(word, 1, len);
    if (stoplist.containsKey(sw)) {
      // assume only simple hyphens (Hyphen.pre="-", Hyphen.post = Hyphen.no =
      // null)
      ArrayList<Object> hw = stoplist.get(sw);
      int j = 0;
      for (i = 0; i < hw.size(); i++) {
        Object o = hw.get(i);
        // j = index(sw) = letterindex(word)?
        // result[k] = corresponding index(w)
        if (o instanceof String) {
          j += ((String) o).length();
          if (j >= remainCharCount && j < (len - pushCharCount)) {
            result[k++] = j + iIgnoreAtBeginning;
          }
        }
      }
    } else {
      // use algorithm to get hyphenation points
      word[0] = '.'; // word start marker
      word[len + 1] = '.'; // word end marker
      word[len + 2] = 0; // null terminated
      byte[] il = new byte[len + 3]; // initialized to zero
      for (i = 0; i < len + 1; i++) {
        searchPatterns(word, i, il);
      }

      // hyphenation points are located where interletter value is odd
      // i is letterindex(word),
      // i + 1 is index(word),
      // result[k] = corresponding index(w)
      for (i = 0; i < len; i++) {
        if (((il[i + 1] & 1) == 1) && i >= remainCharCount
            && i <= (len - pushCharCount)) {
          result[k++] = i + iIgnoreAtBeginning;
        }
      }
    }

    if (k > 0) {
      // trim result array
      int[] res = new int[k+2];
      System.arraycopy(result, 0, res, 1, k);
      // We add the synthetical hyphenation points
      // at the beginning and end of the word
      res[0]=0;
      res[k+1]=len;
      return new Hyphenation(res);
    } else {
      return null;
    }
  }

  /**
   * Add a character class to the tree. It is used by
   * {@link PatternParser PatternParser} as callback to add character classes.
   * Character classes define the valid word characters for hyphenation. If a
   * word contains a character not defined in any of the classes, it is not
   * hyphenated. It also defines a way to normalize the characters in order to
   * compare them with the stored patterns. Usually pattern files use only lower
   * case characters, in this case a class for letter 'a', for example, should
   * be defined as "aA", the first character being the normalization char.
   */
  public void addClass(String chargroup) {
    if (chargroup.length() > 0) {
      char equivChar = chargroup.charAt(0);
      char[] key = new char[2];
      key[1] = 0;
      for (int i = 0; i < chargroup.length(); i++) {
        key[0] = chargroup.charAt(i);
        classmap.insert(key, 0, equivChar);
      }
    }
  }

  /**
   * Add an exception to the tree. It is used by
   * {@link PatternParser PatternParser} class as callback to store the
   * hyphenation exceptions.
   * 
   * @param word normalized word
   * @param hyphenatedword a vector of alternating strings and
   *        {@link Hyphen hyphen} objects.
   */
  public void addException(String word, ArrayList<Object> hyphenatedword) {
    stoplist.put(word, hyphenatedword);
  }

  /**
   * Add a pattern to the tree. Mainly, to be used by
   * {@link PatternParser PatternParser} class as callback to add a pattern to
   * the tree.
   * 
   * @param pattern the hyphenation pattern
   * @param ivalue interletter weight values indicating the desirability and
   *        priority of hyphenating at a given point within the pattern. It
   *        should contain only digit characters. (i.e. '0' to '9').
   */
  public void addPattern(String pattern, String ivalue) {
    int k = ivalues.find(ivalue);
    if (k <= 0) {
      k = packValues(ivalue);
      ivalues.insert(ivalue, (char) k);
    }
    insert(pattern, (char) k);
  }

  @Override
  public void printStats() {
    System.out.println("Value space size = "
        + Integer.toString(vspace.length()));
    super.printStats();

  }
}