Chapter 07
datasets 基本使用
Notebooktransformers61 cells
In [ ]python · cell 1
python
from datasets import *datasets 基本使用
加载在线数据集
In [ ]python · cell 4
python
datasets = load_dataset("madao33/new-title-chinese")
datasets加载数据集合集中的某一项任务
In [ ]python · cell 6
python
boolq_dataset = load_dataset("super_glue", "boolq")
boolq_dataset按照数据集划分进行加载
In [ ]python · cell 8
python
dataset = load_dataset("madao33/new-title-chinese", split="train")
datasetIn [ ]python · cell 9
python
dataset = load_dataset("madao33/new-title-chinese", split="train[10:100]")
datasetIn [ ]python · cell 10
python
dataset = load_dataset("madao33/new-title-chinese", split="train[:50%]")
datasetIn [ ]python · cell 11
python
dataset = load_dataset("madao33/new-title-chinese", split=["train[:50%]", "train[50%:]"])
dataset查看数据集
In [ ]python · cell 13
python
datasets = load_dataset("madao33/new-title-chinese")
datasetsIn [ ]python · cell 14
python
datasets["train"][0]In [ ]python · cell 15
python
datasets["train"][:2]In [ ]python · cell 16
python
datasets["train"]["title"][:5]In [ ]python · cell 17
python
datasets["train"].column_namesIn [ ]python · cell 18
python
datasets["train"].features数据集划分
In [ ]python · cell 20
python
dataset = datasets["train"]
dataset.train_test_split(test_size=0.1)In [ ]python · cell 21
python
dataset = boolq_dataset["train"]
dataset.train_test_split(test_size=0.1, stratify_by_column="label") # 分类数据集可以按照比例划分数据选取与过滤
In [ ]python · cell 23
python
# 选取
datasets["train"].select([0, 1])In [ ]python · cell 24
python
# 过滤
filter_dataset = datasets["train"].filter(lambda example: "中国" in example["title"])In [ ]python · cell 25
python
filter_dataset["title"][:5]数据映射
In [ ]python · cell 27
python
def add_prefix(example):
example["title"] = 'Prefix: ' + example["title"]
return exampleIn [ ]python · cell 28
python
prefix_dataset = datasets.map(add_prefix)
prefix_dataset["train"][:10]["title"]In [ ]python · cell 29
python
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
def preprocess_function(example, tokenizer=tokenizer):
model_inputs = tokenizer(example["content"], max_length=512, truncation=True)
labels = tokenizer(example["title"], max_length=32, truncation=True)
# label就是title编码的结果
model_inputs["labels"] = labels["input_ids"]
return model_inputsIn [ ]python · cell 30
python
processed_datasets = datasets.map(preprocess_function)
processed_datasetsIn [ ]python · cell 31
python
processed_datasets = datasets.map(preprocess_function, num_proc=4)
processed_datasetsIn [ ]python · cell 32
python
processed_datasets = datasets.map(preprocess_function, batched=True)
processed_datasetsIn [ ]python · cell 33
python
processed_datasets = datasets.map(preprocess_function, batched=True, remove_columns=datasets["train"].column_names)
processed_datasets保存与加载
In [ ]python · cell 35
python
processed_datasets.save_to_disk("./processed_data")In [ ]python · cell 36
python
processed_datasets = load_from_disk("./processed_data")
processed_datasets加载本地数据集
直接加载文件作为数据集
In [ ]python · cell 39
python
dataset = load_dataset("csv", data_files="./ChnSentiCorp_htl_all.csv", split="train")
datasetIn [ ]python · cell 40
python
dataset = Dataset.from_csv("./ChnSentiCorp_htl_all.csv")
dataset加载文件夹内全部文件作为数据集
In [ ]python · cell 42
python
dataset = load_dataset("csv", data_files=["./all_data/ChnSentiCorp_htl_all.csv", "./all_data/ChnSentiCorp_htl_all copy.csv"], split='train')
dataset通过预先加载的其他格式转换加载数据集
In [ ]python · cell 44
python
import pandas as pd
data = pd.read_csv("./ChnSentiCorp_htl_all.csv")
data.head()In [ ]python · cell 45
python
dataset = Dataset.from_pandas(data)
datasetIn [ ]python · cell 46
python
# List格式的数据需要内嵌{}，明确数据字段
data = [{"text": "abc"}, {"text": "def"}]
# data = ["abc", "def"]
Dataset.from_list(data)通过自定义加载脚本加载数据集
In [ ]python · cell 48
python
load_dataset("json", data_files="./cmrc2018_trial.json", field="data")In [ ]python · cell 49
python
dataset = load_dataset("./load_script.py", split="train")
datasetIn [ ]python · cell 50
python
dataset[0]Dataset with DataCollator
In [ ]python · cell 52
python
from transformers import DataCollatorWithPaddingIn [ ]python · cell 53
python
dataset = load_dataset("csv", data_files="./ChnSentiCorp_htl_all.csv", split='train')
dataset = dataset.filter(lambda x: x["review"] is not None)
datasetIn [ ]python · cell 54
python
def process_function(examples):
tokenized_examples = tokenizer(examples["review"], max_length=128, truncation=True)
tokenized_examples["labels"] = examples["label"]
return tokenized_examplesIn [ ]python · cell 55
python
tokenized_dataset = dataset.map(process_function, batched=True, remove_columns=dataset.column_names)
tokenized_datasetIn [ ]python · cell 56
python
print(tokenized_dataset[:3])In [ ]python · cell 57
python
collator = DataCollatorWithPadding(tokenizer=tokenizer)In [ ]python · cell 58
python
from torch.utils.data import DataLoaderIn [ ]python · cell 59
python
dl = DataLoader(tokenized_dataset, batch_size=4, collate_fn=collator, shuffle=True)In [ ]python · cell 60
python
num = 0
for batch in dl:
print(batch["input_ids"].size())
num += 1
if num > 10:
breakIn [ ]python · cell 61
python