Class PhonemeTokenizer

Namespace

AiDotNet.Tokenization.Specialized

Assembly

AiDotNet.dll

Phoneme-based tokenizer for speech synthesis (TTS) applications.

public class PhonemeTokenizer : TokenizerBase, ITokenizer

Inheritance

object

TokenizerBase

PhonemeTokenizer

Implements

ITokenizer

Inherited Members

TokenizerBase.Vocabulary

TokenizerBase.SpecialTokens

TokenizerBase.VocabularySize

TokenizerBase.Encode(string, EncodingOptions)

TokenizerBase.EncodeBatch(List<string>, EncodingOptions)

TokenizerBase.Decode(List<int>, bool)

TokenizerBase.DecodeBatch(List<List<int>>, bool)

TokenizerBase.ConvertTokensToIds(List<string>)

TokenizerBase.ConvertIdsToTokens(List<int>)

TokenizerBase.AddSpecialTokensToSequence(List<string>)

TokenizerBase.TruncateSequence(List<string>, int, string)

object.Equals(object)

object.Equals(object, object)

object.GetHashCode()

object.GetType()

object.MemberwiseClone()

object.ReferenceEquals(object, object)

object.ToString()

Constructors

PhonemeTokenizer(IVocabulary, Dictionary<string, string>, SpecialTokens, PhonemeSet)

Creates a new phoneme tokenizer.

public PhonemeTokenizer(IVocabulary vocabulary, Dictionary<string, string> g2pRules, SpecialTokens specialTokens, PhonemeTokenizer.PhonemeSet phonemeSet = PhonemeSet.IPA)

Parameters

vocabulary IVocabulary

g2pRules Dictionary<string, string>

specialTokens SpecialTokens

phonemeSet PhonemeTokenizer.PhonemeSet

Methods

CleanupTokens(List<string>)

Cleans up tokens and converts them back to text (must be implemented by derived classes).

protected override string CleanupTokens(List<string> tokens)

Parameters

tokens List<string>

Returns

string

CreateARPAbet(SpecialTokens?)

Creates a phoneme tokenizer with ARPAbet phonemes.

public static PhonemeTokenizer CreateARPAbet(SpecialTokens? specialTokens = null)

Parameters

specialTokens SpecialTokens

Returns

PhonemeTokenizer

Tokenize(string)

Tokenizes text into phonemes.

public override List<string> Tokenize(string text)

Parameters

text string

Returns

List<string>