BertTokenizer
ApplyBasicTokenization
ClassificationToken
ClassificationTokenId
IndividuallyTokenizeCjk
LowerCaseBeforeTokenization
MaskingToken
MaskingTokenId
PaddingToken
PaddingTokenId
RemoveNonSpacingMarks
SeparatorToken
SeparatorTokenId
SplitOnSpecialTokens
BuildInputsWithSpecialTokens(IEnumerable<Int32>, IEnumerable<Int32>)
BuildInputsWithSpecialTokens(IEnumerable<Int32>, Span<Int32>, Int32, IEnumerable<Int32>)
Create(Stream, BertOptions)
Create(String, BertOptions)
CreateAsync(Stream, BertOptions, CancellationToken)
CreateAsync(String, BertOptions, CancellationToken)
CreateTokenTypeIdsFromSequences(IEnumerable<Int32>, IEnumerable<Int32>)
CreateTokenTypeIdsFromSequences(IEnumerable<Int32>, Span<Int32>, Int32, IEnumerable<Int32>)
EncodeToIds(String, Boolean, Boolean)
EncodeToIds(String, Boolean, Boolean, Boolean)
EncodeToIds(String, Int32, String, Int32, Boolean, Boolean)
EncodeToIds(String, Int32, Boolean, String, Int32, Boolean, Boolean)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Boolean)
EncodeToIds(ReadOnlySpan<Char>, Int32, String, Int32, Boolean, Boolean)
EncodeToIds(ReadOnlySpan<Char>, Int32, Boolean, String, Int32, Boolean, Boolean)
GetSpecialTokensMask(IEnumerable<Int32>, IEnumerable<Int32>, Boolean)
GetSpecialTokensMask(IEnumerable<Int32>, Span<Int32>, Int32, IEnumerable<Int32>, Boolean)
netcoreapp2.0
namespace Microsoft.ML.Tokenizers
{
public sealed class BertTokenizer : WordPieceTokenizer
{
public int SeparatorTokenId { get; }
}
}
.NET | 5.06.07.08.09.010.0 |
---|---|
.NET Core | 2.02.12.23.03.1 |
.NET Framework | 4.6.14.6.24.74.7.14.7.24.84.8.1 |
.NET Standard | 2.02.1 |
Information specific to netcoreapp2.0 | |
Assemblies | Microsoft.ML.Tokenizers , Version=1.0.0.0, PublicKeyToken=cc7b13ffcd2ddd51 Microsoft.ML.Tokenizers , Version=1.0.0.0, PublicKeyToken=cc7b13ffcd2ddd51 |
Referencing | Your project needs a package reference to |
Package | Microsoft.ML.Tokenizers (1.0.2) netstandard2.0 |
Platform Restrictions | This framework does not have platform annotations. |
- Built-in API
- Package-provided API