namespace Microsoft.ML.Tokenizers
{
    public abstract class Tokenizer
    {
        protected virtual int GetIndexByTokenCount(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedText, out int tokenCount);
    }
}