Chapter 03
Tokenizer 基本使用
Notebooktransformers51 cells
Tokenizer 基本使用
In [ ]python · cell 2
python
from transformers import AutoTokenizerIn [ ]python · cell 3
python
sen = "弱小的我也有大梦想!"Step1 加载与保存
In [ ]python · cell 5
python
# 从HuggingFace加载，输入模型名称，即可加载对于的分词器
tokenizer = AutoTokenizer.from_pretrained("uer/roberta-base-finetuned-dianping-chinese")
tokenizerIn [ ]python · cell 6
python
# tokenizer 保存到本地
tokenizer.save_pretrained("./roberta_tokenizer")In [ ]python · cell 7
python
# 从本地加载tokenizer
tokenizer = AutoTokenizer.from_pretrained("./roberta_tokenizer/")
tokenizerStep2 句子分词
In [ ]python · cell 9
python
tokens = tokenizer.tokenize(sen)
tokensStep3 查看词典
In [ ]python · cell 11
python
tokenizer.vocabIn [ ]python · cell 12
python
tokenizer.vocab_sizeStep4 索引转换
In [ ]python · cell 14
python
# 将词序列转换为id序列
ids = tokenizer.convert_tokens_to_ids(tokens)
idsIn [ ]python · cell 15
python
# 将id序列转换为token序列
tokens = tokenizer.convert_ids_to_tokens(ids)
tokensIn [ ]python · cell 16
python
# 将token序列转换为string
str_sen = tokenizer.convert_tokens_to_string(tokens)
str_sen更便捷的实现方式
In [ ]python · cell 18
python
# 将字符串转换为id序列，又称之为编码
ids = tokenizer.encode(sen, add_special_tokens=True)
idsIn [ ]python · cell 19
python
# 将id序列转换为字符串，又称之为解码
str_sen = tokenizer.decode(ids, skip_special_tokens=False)
str_senStep5 填充与截断
In [ ]python · cell 21
python
# 填充
ids = tokenizer.encode(sen, padding="max_length", max_length=15)
idsIn [ ]python · cell 22
python
# 截断
ids = tokenizer.encode(sen, max_length=5, truncation=True)
idsStep6 其他输入部分
In [ ]python · cell 24
python
ids = tokenizer.encode(sen, padding="max_length", max_length=15)
idsIn [ ]python · cell 25
python
attention_mask = [1 if idx != 0 else 0 for idx in ids]
token_type_ids = [0] * len(ids)
ids, attention_mask, token_type_idsStep7 快速调用方式
In [ ]python · cell 27
python
inputs = tokenizer.encode_plus(sen, padding="max_length", max_length=15)
inputsIn [ ]python · cell 28
python
inputs = tokenizer(sen, padding="max_length", max_length=15)
inputsStep8 处理batch数据
In [ ]python · cell 30
python
sens = ["弱小的我也有大梦想",
"有梦想谁都了不起",
"追逐梦想的心，比梦想本身，更可贵"]
res = tokenizer(sens)
resIn [ ]python · cell 31
python
%%time
# 单条循环处理
for i in range(1000):
tokenizer(sen)In [ ]python · cell 32
python
%%time
# 处理batch数据
res = tokenizer([sen] * 1000)In [ ]python · cell 33
python
tokenizerFast / Slow Tokenizer
In [ ]python · cell 35
python
sen = "弱小的我也有大Dreaming!"In [ ]python · cell 36
python
fast_tokenizer = AutoTokenizer.from_pretrained("uer/roberta-base-finetuned-dianping-chinese")
fast_tokenizerIn [ ]python · cell 37
python
slow_tokenizer = AutoTokenizer.from_pretrained("uer/roberta-base-finetuned-dianping-chinese", use_fast=False)
slow_tokenizerIn [ ]python · cell 38
python
%%time
# 单条循环处理
for i in range(10000):
fast_tokenizer(sen)In [ ]python · cell 39
python
%%time
# 单条循环处理
for i in range(10000):
slow_tokenizer(sen)In [ ]python · cell 40
python
%%time
# 处理batch数据
res = fast_tokenizer([sen] * 10000)In [ ]python · cell 41
python
%%time
# 处理batch数据
res = slow_tokenizer([sen] * 10000)In [ ]python · cell 42
python
inputs = fast_tokenizer(sen, return_offsets_mapping=True)
inputsIn [ ]python · cell 43
python
inputs.word_ids()In [ ]python · cell 44
python
inputs = slow_tokenizer(sen, return_offsets_mapping=True)特殊Tokenizer的加载
In [ ]python · cell 46
python
from transformers import AutoTokenizerIn [ ]python · cell 47
python
# 新版本的transformers（>4.34），加载 THUDM/chatglm 会报错，因此这里替换为了天宫的模型
tokenizer = AutoTokenizer.from_pretrained("Skywork/Skywork-13B-base", trust_remote_code=True)
tokenizerIn [ ]python · cell 48
python
tokenizer.save_pretrained("skywork_tokenizer")In [ ]python · cell 49
python
tokenizer = AutoTokenizer.from_pretrained("skywork_tokenizer", trust_remote_code=True)In [ ]python · cell 50
python
tokenizer.decode(tokenizer.encode(sen))In [ ]python · cell 51
python