Class Tokenizer

Namespace

StepLang.Tokenizing

Assembly

StepLang.dll

public class Tokenizer

Inheritance

object

Tokenizer

Inherited Members

object.GetType()

object.MemberwiseClone()

object.ToString()

object.Equals(object)

object.Equals(object, object)

object.ReferenceEquals(object, object)

object.GetHashCode()

Constructors

Tokenizer(CharacterSource, bool)

public Tokenizer(CharacterSource source, bool strict = true)

Parameters

source CharacterSource

strict bool

Methods

Tokenize(CancellationToken)

public IEnumerable<Token> Tokenize(CancellationToken cancellationToken = default)

Parameters

cancellationToken CancellationToken

Returns

IEnumerable<Token>