SentencePieceTokenizer
AddBeginningOfSentence
AddDummyPrefix
AddEndOfSentence
BeginningOfSentenceId
BeginningOfSentenceToken
ByteFallback
EndOfSentenceId
EndOfSentenceToken
EscapeWhiteSpaces
Normalizer
PreTokenizer
SpecialTokens
TreatWhitespaceAsSuffix
UnknownId
UnknownToken
Vocabulary
CountTokens(String, Boolean, Boolean, Boolean, Boolean)
CountTokens(String, Boolean, Boolean, Boolean, Boolean, String, Int32, Int32)
CountTokens(String, ReadOnlySpan<Char>, EncodeSettings)
CountTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean)
CountTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean, String, Int32, Int32)
Create(IEnumerable<(String Piece, Single Score)>, Int32, Boolean, Boolean, ReadOnlySpan<Byte>, Boolean, Boolean, Boolean, Boolean, IReadOnlyDictionary<String, Int32>)
Create(Stream, Boolean, Boolean, IReadOnlyDictionary<String, Int32>)
CreateFromTokenizerJson(Stream, Boolean, Boolean, IReadOnlyDictionary<String, Int32>)
Decode(IEnumerable<Int32>)
Decode(IEnumerable<Int32>, Boolean)
Decode(IEnumerable<Int32>, Span<Char>, Int32, Int32)
Decode(IEnumerable<Int32>, Span<Char>, Boolean, Int32, Int32)
EncodeToIds(String, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(String, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
EncodeToIds(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
EncodeToTokens(String, String, Boolean, Boolean, Boolean, Boolean)
EncodeToTokens(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToTokens(ReadOnlySpan<Char>, String, Boolean, Boolean, Boolean, Boolean)
GetIndexByTokenCount(String, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
GetIndexByTokenCount(String, ReadOnlySpan<Char>, EncodeSettings, Boolean, String, Int32)
GetIndexByTokenCount(ReadOnlySpan<Char>, Boolean, Boolean, Int32, String, Int32, Boolean, Boolean)
GetIndexByTokenCountFromEnd(String, Boolean, Boolean, Int32, Boolean, String, Int32)
GetIndexByTokenCountFromEnd(ReadOnlySpan<Char>, Boolean, Boolean, Int32, Boolean, String, Int32)
monoandroid
namespace Microsoft.ML.Tokenizers
{
public class SentencePieceTokenizer : Tokenizer
{
public int EndOfSentenceId { get; }
}
}
| .NET | 5.06.07.08.09.011.0 |
|---|---|
| .NET Core | 2.02.12.23.03.1 |
| .NET Framework | 4.6.14.6.24.74.7.14.7.24.84.8.1 |
| .NET Standard | 2.02.1 |
Information specific to monoandroid | |
| Assembly | Microsoft.ML.Tokenizers , Version=1.0.0.0, PublicKeyToken=cc7b13ffcd2ddd51 |
| Referencing | Your project needs a package reference to |
| Package | Microsoft.ML.Tokenizers (3.0.0) netstandard2.0 |
| Platform Restrictions | This framework does not have platform annotations. |
- Built-in API
- Package-provided API