namespace Microsoft.ML.Tokenizers
{
    public class SentencePieceTokenizer : Tokenizer
    {
        public int CountTokens(string! text, bool addBeginningOfSentence, bool addEndOfSentence, bool considerPreTokenization, bool considerNormalization, out string? normalizedText, out int charsConsumed, int maxTokenCount = 2147483647);
    }
}