namespace Microsoft.ML.Tokenizers
{
    public class CodeGenTokenizer : Tokenizer
    {
        public int GetIndexByTokenCountFromEnd(string! text, int maxTokenCount, bool addPrefixSpace, bool addBeginningOfSentence, bool addEndOfSentence, out string? normalizedString, out int tokenCount, bool considerPreTokenization = true, bool considerNormalization = true);
    }
}