namespace Microsoft.ML.Tokenizers
{
    public class SentencePieceTokenizer : Tokenizer
    {
        public int GetIndexByTokenCount(ReadOnlySpan<char> text, bool addBeginningOfSentence, bool addEndOfSentence, int maxTokenCount, out string? normalizedText, out int tokenCount, bool considerPreTokenization = true, bool considerNormalization = true);
    }
}