Class SpecialTokens

Namespace

AiDotNet.Tokenization.Models

Assembly

AiDotNet.dll

Represents special tokens used by tokenizers.

public class SpecialTokens

Inheritance

object

SpecialTokens

Inherited Members

object.Equals(object)

object.Equals(object, object)

object.GetHashCode()

object.GetType()

object.MemberwiseClone()

object.ReferenceEquals(object, object)

object.ToString()

Properties

AdditionalSpecialTokens

Gets or sets additional special tokens.

public List<string> AdditionalSpecialTokens { get; set; }

Property Value

List<string>

BosToken

Gets or sets the beginning of sequence token.

public string BosToken { get; set; }

Property Value

string

ClsToken

Gets or sets the classification token (start of sequence).

public string ClsToken { get; set; }

Property Value

string

EosToken

Gets or sets the end of sequence token.

public string EosToken { get; set; }

Property Value

string

MaskToken

Gets or sets the mask token.

public string MaskToken { get; set; }

Property Value

string

PadToken

Gets or sets the padding token.

public string PadToken { get; set; }

Property Value

string

SepToken

Gets or sets the separation token.

public string SepToken { get; set; }

Property Value

string

UnkToken

Gets or sets the unknown token.

public string UnkToken { get; set; }

Property Value

string

Methods

Bert()

Creates BERT-style special tokens.

public static SpecialTokens Bert()

Returns

SpecialTokens

Clip()

public static SpecialTokens Clip()

Returns

SpecialTokens

Default()

Creates default special tokens (BERT-style).

public static SpecialTokens Default()

Returns

SpecialTokens

GetAllSpecialTokens()

Gets all special tokens as a list.

public List<string> GetAllSpecialTokens()

Returns

List<string>

Gpt()

Creates GPT-style special tokens.

public static SpecialTokens Gpt()

Returns

SpecialTokens

T5()

Creates T5-style special tokens.

public static SpecialTokens T5()

Returns

SpecialTokens