|
Java example source code file (BaseTextVectorizer.java)
The BaseTextVectorizer.java Java example source codepackage org.deeplearning4j.bagofwords.vectorizer; import lombok.Getter; import lombok.Setter; import org.deeplearning4j.models.sequencevectors.iterators.AbstractSequenceIterator; import org.deeplearning4j.models.sequencevectors.transformers.impl.SentenceTransformer; import org.deeplearning4j.models.word2vec.VocabWord; import org.deeplearning4j.models.word2vec.wordstore.VocabCache; import org.deeplearning4j.models.word2vec.wordstore.VocabConstructor; import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache; import org.deeplearning4j.text.documentiterator.LabelAwareIterator; import org.deeplearning4j.text.documentiterator.LabelsSource; import org.deeplearning4j.text.invertedindex.InvertedIndex; import org.deeplearning4j.text.sentenceiterator.SentenceIterator; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; import java.util.ArrayList; import java.util.List; /** * @author raver119@gmail.com */ public abstract class BaseTextVectorizer implements TextVectorizer { @Setter protected transient TokenizerFactory tokenizerFactory; protected transient LabelAwareIterator iterator; protected int minWordFrequency; @Getter protected VocabCache<VocabWord> vocabCache; protected LabelsSource labelsSource; protected List<String> stopWords = new ArrayList<>(); @Getter protected transient InvertedIndex<VocabWord> index; protected LabelsSource getLabelsSource() { return labelsSource; } public void buildVocab() { if (vocabCache == null) vocabCache = new AbstractCache.Builder<VocabWord>().build(); SentenceTransformer transformer = new SentenceTransformer.Builder() .iterator(this.iterator) .tokenizerFactory(tokenizerFactory) .build(); AbstractSequenceIterator<VocabWord> iterator = new AbstractSequenceIterator.Builder<>(transformer) .build(); VocabConstructor<VocabWord> constructor = new VocabConstructor.Builder Other Java examples (source code examples)Here is a short list of links related to this Java BaseTextVectorizer.java source code file: |
... this post is sponsored by my books ... | |
#1 New Release! |
FP Best Seller |
Copyright 1998-2021 Alvin Alexander, alvinalexander.com
All Rights Reserved.
A percentage of advertising revenue from
pages under the /java/jwarehouse
URI on this website is
paid back to open source projects.