namespace Microsoft.ML.Tokenizers
{
    public abstract class Tokenizer
    {
        public int GetIndexByTokenCountFromEnd(ReadOnlySpan<char> text, int maxTokenCount, out string? normalizedText, out int tokenCount, bool considerPreTokenization = true, bool considerNormalization = true);
    }
}