SentencePieceTokenizer
AddBeginningOfSentence
AddDummyPrefix
AddEndOfSentence
BeginningOfSentenceId
BeginningOfSentenceToken
ByteFallback
EndOfSentenceId
EndOfSentenceToken
EscapeWhiteSpaces
Normalizer
PreTokenizer
SpecialTokens
TreatWhitespaceAsSuffix
UnknownId
UnknownToken
Vocabulary
CountTokens(String, Boolean, Boolean, Boolean, Boolean)
CountTokens(String, Boolean, Boolean, Boolean, Boolean, String, Int32, Int32)
CountTokens(String, ReadOnlySpan<Char>, EncodeSettings)
CountTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean)
CountTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean, String, Int32, Int32)
Create(SentencePieceOptions)
Create(Stream, Boolean, Boolean, IReadOnlyDictionary<String, Int32>)
Decode(IEnumerable<Int32>)
Decode(IEnumerable<Int32>, Boolean)
Decode(IEnumerable<Int32>, Span<Char>, Int32, Int32)
Decode(IEnumerable<Int32>, Span<Char>, Boolean, Int32, Int32)
EncodeToIds(String, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(String, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
EncodeToIds(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
EncodeToTokens(String, String, Boolean, Boolean, Boolean, Boolean)
EncodeToTokens(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToTokens(ReadOnlySpan<Char>, String, Boolean, Boolean, Boolean, Boolean)
GetIndexByTokenCount(String, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
GetIndexByTokenCount(String, ReadOnlySpan<Char>, EncodeSettings, Boolean, String, Int32)
GetIndexByTokenCount(ReadOnlySpan<Char>, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
GetIndexByTokenCountFromEnd(String, Boolean, Boolean, Int32, Boolean, String, Int32)
GetIndexByTokenCountFromEnd(ReadOnlySpan<Char>, Boolean, Boolean, Int32, Boolean, String, Int32)
netcoreapp3.1
namespace Microsoft.ML.Tokenizers
{
public class SentencePieceTokenizer : Tokenizer
{
public static SentencePieceTokenizer! Create(SentencePieceOptions! options);
}
}
.NET | 5.06.07.08.09.010.0 |
---|---|
.NET Core | 2.02.12.23.03.1 |
.NET Framework | 4.6.14.6.24.74.7.14.7.24.84.8.1 |
.NET Standard | 2.02.1 |
Information specific to netcoreapp3.1 | |
Assembly | Microsoft.ML.Tokenizers , Version=1.0.0.0, PublicKeyToken=cc7b13ffcd2ddd51 |
Referencing | Your project needs a package reference to |
Package | Microsoft.ML.Tokenizers (2.0.0-preview.1.25127.4) netstandard2.0 |
Preview | This API is contained in a prerelease package. |
Platform Restrictions | This framework does not have platform annotations. |
- Built-in API
- Package-provided API