SentencePieceTokenizer
AddBeginningOfSentence
AddDummyPrefix
AddEndOfSentence
BeginningOfSentenceId
BeginningOfSentenceToken
ByteFallback
EndOfSentenceId
EndOfSentenceToken
EscapeWhiteSpaces
Normalizer
PreTokenizer
SpecialTokens
TreatWhitespaceAsSuffix
UnknownId
UnknownToken
Vocabulary
CountTokens(String, Boolean, Boolean, Boolean, Boolean)
CountTokens(String, Boolean, Boolean, Boolean, Boolean, String, Int32, Int32)
CountTokens(String, ReadOnlySpan<Char>, EncodeSettings)
CountTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean)
CountTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean, String, Int32, Int32)
Create(SentencePieceOptions)
Create(Stream, Boolean, Boolean, IReadOnlyDictionary<String, Int32>)
Decode(IEnumerable<Int32>)
Decode(IEnumerable<Int32>, Boolean)
Decode(IEnumerable<Int32>, Span<Char>, Int32, Int32)
Decode(IEnumerable<Int32>, Span<Char>, Boolean, Int32, Int32)
EncodeToIds(String, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(String, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
EncodeToIds(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
EncodeToTokens(String, String, Boolean, Boolean, Boolean, Boolean)
EncodeToTokens(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToTokens(ReadOnlySpan<Char>, String, Boolean, Boolean, Boolean, Boolean)
GetIndexByTokenCount(String, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
GetIndexByTokenCount(String, ReadOnlySpan<Char>, EncodeSettings, Boolean, String, Int32)
GetIndexByTokenCount(ReadOnlySpan<Char>, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
GetIndexByTokenCountFromEnd(String, Boolean, Boolean, Int32, Boolean, String, Int32)
GetIndexByTokenCountFromEnd(ReadOnlySpan<Char>, Boolean, Boolean, Int32, Boolean, String, Int32)
net10.0-windows7.0
namespace Microsoft.ML.Tokenizers
{
public class SentencePieceTokenizer : Tokenizer
{
protected override EncodeResults<int> EncodeToIds(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings);
}
}
.NET | 5.06.07.08.09.010.0 |
---|---|
.NET Core | 2.02.12.23.03.1 |
.NET Framework | 4.6.14.6.24.74.7.14.7.24.84.8.1 |
.NET Standard | 2.02.1 |
Information specific to net10.0 | |
Platforms | This API is only available when you target a specific platform: |
Windows | 7.0 |
Information specific to net10.0-windows7.0 | |
Assemblies | Microsoft.ML.Tokenizers , Version=1.0.0.0, PublicKeyToken=cc7b13ffcd2ddd51 Microsoft.ML.Tokenizers , Version=1.0.0.0, PublicKeyToken=cc7b13ffcd2ddd51 |
Referencing | Your project needs a package reference to |
Package | Microsoft.ML.Tokenizers (2.0.0-preview.1.25127.4) net8.0 |
Platform Restrictions | This API is supported on all platforms. |
- Built-in API
- Package-provided API