SentencePieceTokenizer
AddBeginningOfSentence
AddDummyPrefix
AddEndOfSentence
BeginningOfSentenceId
BeginningOfSentenceToken
ByteFallback
EndOfSentenceId
EndOfSentenceToken
EscapeWhiteSpaces
Normalizer
PreTokenizer
SpecialTokens
TreatWhitespaceAsSuffix
UnknownId
UnknownToken
Vocabulary
CountTokens(String, Boolean, Boolean, Boolean, Boolean)
CountTokens(String, Boolean, Boolean, Boolean, Boolean, String, Int32, Int32)
CountTokens(String, ReadOnlySpan<Char>, EncodeSettings)
CountTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean)
CountTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean, String, Int32, Int32)
Create(SentencePieceOptions)
Create(Stream, Boolean, Boolean, IReadOnlyDictionary<String, Int32>)
Decode(IEnumerable<Int32>)
Decode(IEnumerable<Int32>, Boolean)
Decode(IEnumerable<Int32>, Span<Char>, Int32, Int32)
Decode(IEnumerable<Int32>, Span<Char>, Boolean, Int32, Int32)
EncodeToIds(String, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(String, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
EncodeToIds(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
EncodeToTokens(String, String, Boolean, Boolean, Boolean, Boolean)
EncodeToTokens(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToTokens(ReadOnlySpan<Char>, String, Boolean, Boolean, Boolean, Boolean)
GetIndexByTokenCount(String, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
GetIndexByTokenCount(String, ReadOnlySpan<Char>, EncodeSettings, Boolean, String, Int32)
GetIndexByTokenCount(ReadOnlySpan<Char>, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
GetIndexByTokenCountFromEnd(String, Boolean, Boolean, Int32, Boolean, String, Int32)
GetIndexByTokenCountFromEnd(ReadOnlySpan<Char>, Boolean, Boolean, Int32, Boolean, String, Int32)
net10.0-windows7.0
namespace Microsoft.ML.Tokenizers
{
public class SentencePieceTokenizer : Tokenizer
{
public static SentencePieceTokenizer! Create(SentencePieceOptions! options);
}
}
.NET | 5.06.07.08.09.010.0 |
---|---|
.NET Core | 2.02.12.23.03.1 |
.NET Framework | 4.6.14.6.24.74.7.14.7.24.84.8.1 |
.NET Standard | 2.02.1 |
Information specific to net10.0 | |
Platforms | This API is only available when you target a specific platform: |
Windows | 7.0 |
Information specific to net10.0-windows7.0 | |
Assembly | Microsoft.ML.Tokenizers , Version=1.0.0.0, PublicKeyToken=cc7b13ffcd2ddd51 |
Referencing | Your project needs a package reference to |
Package | Microsoft.ML.Tokenizers (2.0.0-preview.1.25127.4) net8.0 |
Preview | This API is contained in a prerelease package. |
Platform Restrictions | This API is supported on all platforms. |
- Built-in API
- Package-provided API