Java example source code file (ContextLabelRetriever.java)

This example Java source code file (ContextLabelRetriever.java) is included in the alvinalexander.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Learn more about this Java project at its project page.

Java - Java tags/keywords

a-za-z, arraylist, contextlabelretriever, current, end_label, found, illegalstateexception, list, multidimensionalmap, none, pair, string, stringbuffer, util, was

The ContextLabelRetriever.java Java example source code

/*
 *
 *  * Copyright 2015 Skymind,Inc.
 *  *
 *  *    Licensed under the Apache License, Version 2.0 (the "License");
 *  *    you may not use this file except in compliance with the License.
 *  *    You may obtain a copy of the License at
 *  *
 *  *        http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *    Unless required by applicable law or agreed to in writing, software
 *  *    distributed under the License is distributed on an "AS IS" BASIS,
 *  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *    See the License for the specific language governing permissions and
 *  *    limitations under the License.
 *
 */

package org.deeplearning4j.text.movingwindow;

import org.deeplearning4j.berkeley.Pair;
import org.deeplearning4j.berkeley.StringUtils;
import org.deeplearning4j.util.MultiDimensionalMap;
import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;

import java.util.ArrayList;
import java.util.List;

/**
 * Context Label Retriever
 * @author Adam Gibson
 */
public class ContextLabelRetriever {


    private static String BEGIN_LABEL = "<([A-Za-z]+|\\d+)>";
    private static String END_LABEL = "</([A-Za-z]+|\\d+)>";


   private ContextLabelRetriever() {}


    /**
     * Returns a stripped sentence with the indices of words
     * with certain kinds of labels.
     * @param sentence the sentence to process
     * @return a pair of a post processed sentence
     * with labels stripped and the spans of
     * the labels
     */
    public static Pair<String,MultiDimensionalMap stringWithLabels(String sentence,TokenizerFactory tokenizerFactory) {
        MultiDimensionalMap<Integer,Integer,String> map = MultiDimensionalMap.newHashBackedMap();
        Tokenizer t = tokenizerFactory.create(sentence);
        List<String> currTokens = new ArrayList<>();
        String currLabel = null;
        String endLabel = null;
        List<Pair> tokensWithSameLabel = new ArrayList<>();
        while(t.hasMoreTokens()) {
            String token = t.nextToken();
            if(token.matches(BEGIN_LABEL)) {
                if(endLabel != null)
                    throw new IllegalStateException("Tried parsing sentence; found an end label when the begin label has not been cleared");
                currLabel = token;

                //no labels; add these as NONE and begin the new label
                if(!currTokens.isEmpty()) {
                    tokensWithSameLabel.add(new Pair<>("NONE",(List) new ArrayList<>(currTokens)));
                    currTokens.clear();

                }

            }
            else if(token.matches(END_LABEL)) {
                if(currLabel ==  null)
                    throw new IllegalStateException("Found an ending label with no matching begin label");
                endLabel = token;
            }
            else
                currTokens.add(token);

            if(currLabel != null && endLabel != null) {
                currLabel = currLabel.replaceAll("[<>/]","");
                endLabel = endLabel.replaceAll("[<>/]","");
                assert !currLabel.isEmpty() : "Current label is empty!";
                assert !endLabel.isEmpty() : "End label is empty!";
                assert currLabel.equals(endLabel) : "Current label begin and end did not match for the parse. Was: " + currLabel + " ending with " + endLabel;

                tokensWithSameLabel.add(new Pair<>(currLabel,(List) new ArrayList<>(currTokens)));
                currTokens.clear();


                //clear out the tokens
                currLabel = null;
                endLabel = null;
            }


        }

        //no labels; add these as NONE and begin the new label
        if(!currTokens.isEmpty()) {
            tokensWithSameLabel.add(new Pair<>("none",(List) new ArrayList<>(currTokens)));
            currTokens.clear();

        }

        //now join the output
        StringBuffer strippedSentence = new StringBuffer();
        for(Pair<String,List tokensWithLabel : tokensWithSameLabel) {
            String joinedSentence = StringUtils.join(tokensWithLabel.getSecond()," ");
            //spaces between separate parts of the sentence
            if(!(strippedSentence.length() < 1))
                strippedSentence.append(" ");
            strippedSentence.append(joinedSentence);
            int begin = strippedSentence.toString().indexOf(joinedSentence);
            int end = begin + joinedSentence.length();
            map.put(begin,end,tokensWithLabel.getFirst());
        }



        return new Pair<>(strippedSentence.toString(),map);
    }



}