Lucene example source code file (ExtractReuters.java)

This example Lucene source code file (ExtractReuters.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Java - Lucene tags/keywords

bufferedreader, extractreuters, extractreuters, file, file, filewriter, io, ioexception, meta_chars, pattern, regex, reuters, string, string, stringbuilder, stringbuilder

The Lucene ExtractReuters.java source code

package org.apache.lucene.benchmark.utils;

/**
 * Copyright 2005 The Apache Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Split the Reuters SGML documents into Simple Text files containing: Title,
 * Date, Dateline, Body
 */
public class ExtractReuters {
  private File reutersDir;
  private File outputDir;
  private static final String LINE_SEPARATOR = System.getProperty("line.separator");

  public ExtractReuters(File reutersDir, File outputDir) {
    this.reutersDir = reutersDir;
    this.outputDir = outputDir;
    System.out.println("Deleting all files in " + outputDir);
    for (File f : outputDir.listFiles()) {
      f.delete();
    }
  }

  public void extract() {
    File[] sgmFiles = reutersDir.listFiles(new FileFilter() {
      public boolean accept(File file) {
        return file.getName().endsWith(".sgm");
      }
    });
    if (sgmFiles != null && sgmFiles.length > 0) {
      for (File sgmFile : sgmFiles) {
        extractFile(sgmFile);
      }
    } else {
      System.err.println("No .sgm files in " + reutersDir);
    }
  }

  Pattern EXTRACTION_PATTERN = Pattern
      .compile("<TITLE>(.*?)|(.*?)|(.*?)");

  private static String[] META_CHARS = { "&", "<", ">", "\"", "'" };

  private static String[] META_CHARS_SERIALIZATIONS = { "&", "<",
      ">", """, "'" };

  /**
   * Override if you wish to change what is extracted
   * 
   * @param sgmFile
   */
  protected void extractFile(File sgmFile) {
    try {
      BufferedReader reader = new BufferedReader(new FileReader(sgmFile));

      StringBuilder buffer = new StringBuilder(1024);
      StringBuilder outBuffer = new StringBuilder(1024);

      String line = null;
      int docNumber = 0;
      while ((line = reader.readLine()) != null) {
        // when we see a closing reuters tag, flush the file

        if (line.indexOf("</REUTERS") == -1) {
          // Replace the SGM escape sequences

          buffer.append(line).append(' ');// accumulate the strings for now,
                                          // then apply regular expression to
                                          // get the pieces,
        } else {
          // Extract the relevant pieces and write to a file in the output dir
          Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
          while (matcher.find()) {
            for (int i = 1; i <= matcher.groupCount(); i++) {
              if (matcher.group(i) != null) {
                outBuffer.append(matcher.group(i));
              }
            }
            outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
          }
          String out = outBuffer.toString();
          for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) {
            out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
          }
          File outFile = new File(outputDir, sgmFile.getName() + "-"
              + (docNumber++) + ".txt");
          // System.out.println("Writing " + outFile);
          FileWriter writer = new FileWriter(outFile);
          writer.write(out);
          writer.close();
          outBuffer.setLength(0);
          buffer.setLength(0);
        }
      }
      reader.close();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  public static void main(String[] args) {
    if (args.length != 2) {
      usage("Wrong number of arguments ("+args.length+")");
      return;
    }
    File reutersDir = new File(args[0]);
    if (!reutersDir.exists()) {
      usage("Cannot find Path to Reuters SGM files ("+reutersDir+")");
      return;
    }
    
    // First, extract to a tmp directory and only if everything succeeds, rename
    // to output directory.
    File outputDir = new File(args[1]);
    outputDir = new File(outputDir.getAbsolutePath() + "-tmp");
    outputDir.mkdirs();
    ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
    extractor.extract();
    // Now rename to requested output dir
    outputDir.renameTo(new File(args[1]));
  }

  private static void usage(String msg) {
    System.err.println("Usage: "+msg+" :: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters  
");
  }
  
}