CodeGenTokenizer
AddBeginningOfSentence
AddEndOfSentence
AddPrefixSpace
BeginningOfSentenceId
BeginningOfSentenceToken
EndOfSentenceId
EndOfSentenceToken
Normalizer
PreTokenizer
SpecialTokens
UnknownToken
UnknownTokenId
Vocabulary
CountTokens(String, Boolean, Boolean, Boolean, Boolean, Boolean)
CountTokens(String, ReadOnlySpan<Char>, EncodeSettings)
CountTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean, Boolean)
Create(Stream, Stream, Boolean, Boolean, Boolean)
Decode(IEnumerable<Int32>)
Decode(IEnumerable<Int32>, Boolean, Boolean)
Decode(IEnumerable<Int32>, Span<Char>, Int32, Int32)
Decode(IEnumerable<Int32>, Span<Char>, Boolean, Boolean, Int32, Int32)
EncodeToIds(String, Boolean, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(String, Int32, Boolean, Boolean, Boolean, String, Int32, Boolean, Boolean)
EncodeToIds(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToIds(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, Boolean, Boolean)
EncodeToIds(ReadOnlySpan<Char>, Int32, Boolean, Boolean, Boolean, String, Int32, Boolean, Boolean)
EncodeToTokens(String, Boolean, Boolean, Boolean, String, Boolean, Boolean)
EncodeToTokens(String, ReadOnlySpan<Char>, EncodeSettings)
EncodeToTokens(ReadOnlySpan<Char>, Boolean, Boolean, Boolean, String, Boolean, Boolean)
GetIndexByTokenCount(String, Int32, Boolean, Boolean, Boolean, String, Int32, Boolean, Boolean)
GetIndexByTokenCount(String, ReadOnlySpan<Char>, EncodeSettings, Boolean, String, Int32)
GetIndexByTokenCount(ReadOnlySpan<Char>, Int32, Boolean, Boolean, Boolean, String, Int32, Boolean, Boolean)
GetIndexByTokenCountFromEnd(String, Int32, Boolean, Boolean, Boolean, String, Int32, Boolean, Boolean)
GetIndexByTokenCountFromEnd(ReadOnlySpan<Char>, Int32, Boolean, Boolean, Boolean, String, Int32, Boolean, Boolean)
net10.0-windows7.0
namespace Microsoft.ML.Tokenizers
{
public class CodeGenTokenizer : Tokenizer
{
public bool AddPrefixSpace { get; }
}
}
.NET | 5.06.07.08.09.010.0 |
---|---|
.NET Core | 2.02.12.23.03.1 |
.NET Framework | 4.6.14.6.24.74.7.14.7.24.84.8.1 |
.NET Standard | 2.02.1 |
Information specific to net10.0 | |
Platforms | This API is only available when you target a specific platform: |
Windows | 7.0 |
Information specific to net10.0-windows7.0 | |
Assemblies | Microsoft.ML.Tokenizers , Version=1.0.0.0, PublicKeyToken=cc7b13ffcd2ddd51 Microsoft.ML.Tokenizers , Version=1.0.0.0, PublicKeyToken=cc7b13ffcd2ddd51 |
Referencing | Your project needs a package reference to |
Package | Microsoft.ML.Tokenizers (2.0.0-preview.1.25127.4) net8.0 |
Platform Restrictions | The property getter is supported on all platforms. |
- Built-in API
- Package-provided API