namespace Microsoft.ML.Tokenizers
{
    public sealed class TiktokenTokenizer : Tokenizer
    {
        protected override int GetIndexByTokenCount(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedString, out int tokenCount);
    }
}