Class CodeTokenizationPipeline

Namespace

AiDotNet.ProgramSynthesis.Tokenization

Assembly

AiDotNet.dll

Default implementation of ICodeTokenizationPipeline.

public sealed class CodeTokenizationPipeline : ICodeTokenizationPipeline

Inheritance

object

CodeTokenizationPipeline

Implements

ICodeTokenizationPipeline

Inherited Members

object.Equals(object)

object.Equals(object, object)

object.GetHashCode()

object.GetType()

object.ReferenceEquals(object, object)

object.ToString()

Methods

Tokenize(string, ProgramLanguage, ITokenizer, EncodingOptions?, string?)

public CodeTokenizationResult Tokenize(string code, ProgramLanguage language, ITokenizer tokenizer, EncodingOptions? options = null, string? filePath = null)

Parameters

code string

language ProgramLanguage

tokenizer ITokenizer

options EncodingOptions

filePath string

Returns

CodeTokenizationResult

TokenizeWithStructure(string, ProgramLanguage, ITokenizer, CodeTokenizationPipelineOptions, EncodingOptions?, string?)

public CodeTokenizationResult TokenizeWithStructure(string code, ProgramLanguage language, ITokenizer tokenizer, CodeTokenizationPipelineOptions pipelineOptions, EncodingOptions? options = null, string? filePath = null)

Parameters

code string

language ProgramLanguage

tokenizer ITokenizer

pipelineOptions CodeTokenizationPipelineOptions

options EncodingOptions

filePath string

Returns

CodeTokenizationResult