namespace Microsoft.ML.Tokenizers
{
    public class WordPieceTokenizer : Tokenizer
    {
        [AsyncStateMachine(typeof(<CreateAsync>d__21))]
        public static Task<WordPieceTokenizer!>! CreateAsync(Stream! vocabStream, PreTokenizer? preTokenizer = null, Normalizer? normalizer = null, IReadOnlyDictionary<string!, int>? specialTokens = null, string! unknownToken = "[UNK]", string! continuingSubwordPrefix = "##", int maxInputCharsPerWord = 100, CancellationToken cancellationToken = default);
    }
}