Class UnigramTokenizer

Unigram Language Model tokenizer using probabilistic segmentation.

public class UnigramTokenizer : TokenizerBase, ITokenizer

Inherited Members: TokenizerBase.Vocabulary

TokenizerBase.SpecialTokens

TokenizerBase.VocabularySize

TokenizerBase.Encode(string, EncodingOptions)

TokenizerBase.EncodeBatch(List<string>, EncodingOptions)

TokenizerBase.Decode(List<int>, bool)

TokenizerBase.DecodeBatch(List<List<int>>, bool)

TokenizerBase.ConvertTokensToIds(List<string>)

TokenizerBase.ConvertIdsToTokens(List<int>)

TokenizerBase.AddSpecialTokensToSequence(List<string>)

TokenizerBase.TruncateSequence(List<string>, int, string)

object.Equals(object)

object.Equals(object, object)

object.GetHashCode()

object.GetType()

object.MemberwiseClone()

object.ReferenceEquals(object, object)

object.ToString()

Constructors

Creates a new unigram tokenizer.

public UnigramTokenizer(IVocabulary vocabulary, Dictionary<string, double> tokenScores, SpecialTokens specialTokens, int maxTokenLength = 16)

Cleans up tokens and converts them back to text (must be implemented by derived classes).

protected override string CleanupTokens(List<string> tokens)

Tokenizes text using Viterbi algorithm for optimal segmentation.

public override List<string> Tokenize(string text)

Trains a unigram tokenizer from a corpus.

public static UnigramTokenizer Train(IEnumerable<string> corpus, int vocabSize = 8000, SpecialTokens? specialTokens = null)