利用预训练模型SKEP进行情感分析

!pip install --upgrade paddlenlp -i https://pypi.org/simple 

from paddlenlp.datasets import load_datasettrain_ds, dev_ds, test_ds = load_dataset("chnsenticorp", splits=["train", "dev", "test"])print(train_ds[0])
print(train_ds[1])
print(train_ds[2])

from paddlenlp.transformers import SkepForSequenceClassification, SkepTokenizer# 指定模型名称，一键加载模型
model = SkepForSequenceClassification.from_pretrained(pretrained_model_name_or_path="skep_ernie_1.0_large_ch", num_classes=len(train_ds.label_list))
# 同样地，通过指定模型名称一键加载对应的Tokenizer，用于处理文本数据，如切分token，转token_id等。
tokenizer = SkepTokenizer.from_pretrained(pretrained_model_name_or_path="skep_ernie_1.0_large_ch")

import os
from functools import partialimport numpy as np
import paddle
import paddle.nn.functional as F
from paddlenlp.data import Stack, Tuple, Padfrom utils import create_dataloaderdef convert_example(example,tokenizer,max_seq_length=512,is_test=False):"""Builds model inputs from a sequence or a pair of sequence for sequence classification tasksby concatenating and adding special tokens. And creates a mask from the two sequences passed to be used in a sequence-pair classification task.A skep_ernie_1.0_large_ch/skep_ernie_2.0_large_en sequence has the following format:::- single sequence: ``[CLS] X [SEP]``- pair of sequences: ``[CLS] A [SEP] B [SEP]``A skep_ernie_1.0_large_ch/skep_ernie_2.0_large_en sequence pair mask has the following format:::0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1| first sequence    | second sequence |If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).Args:example(obj:`list[str]`): List of input data, containing text and label if it have label.tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer` which contains most of the methods. Users should refer to the superclass for more information regarding methods.max_seq_len(obj:`int`): The maximum total input sequence length after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded.is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.Returns:input_ids(obj:`list[int]`): The list of token ids.token_type_ids(obj: `list[int]`): List of sequence pair mask.label(obj:`int`, optional): The input label if not is_test."""# 将原数据处理成model可读入的格式，enocded_inputs是一个dict，包含input_ids、token_type_ids等字段encoded_inputs = tokenizer(text=example["text"], max_seq_len=max_seq_length)# input_ids：对文本切分token后，在词汇表中对应的token idinput_ids = encoded_inputs["input_ids"]# token_type_ids：当前token属于句子1还是句子2，即上述图中表达的segment idstoken_type_ids = encoded_inputs["token_type_ids"]if not is_test:# label：情感极性类别label = np.array([example["label"]], dtype="int64")return input_ids, token_type_ids, labelelse:# qid：每条数据的编号qid = np.array([example["qid"]], dtype="int64")return input_ids, token_type_ids, qid

# 批量数据大小
batch_size = 32
# 文本序列最大长度
max_seq_length = 128# 将数据处理成模型可读入的数据格式
trans_func = partial(convert_example,tokenizer=tokenizer,max_seq_length=max_seq_length)# 将数据组成批量式数据，如
# 将不同长度的文本序列padding到批量式数据中最大长度
# 将每条数据label堆叠在一起
batchify_fn = lambda samples, fn=Tuple(Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input_idsPad(axis=0, pad_val=tokenizer.pad_token_type_id),  # token_type_idsStack()  # labels
): [data for data in fn(samples)]
train_data_loader = create_dataloader(train_ds,mode='train',batch_size=batch_size,batchify_fn=batchify_fn,trans_fn=trans_func)
dev_data_loader = create_dataloader(dev_ds,mode='dev',batch_size=batch_size,batchify_fn=batchify_fn,trans_fn=trans_func)

# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
import paddledef create_dataloader(dataset,trans_fn=None,mode='train',batch_size=1,batchify_fn=None):"""Creats dataloader.Args:dataset(obj:`paddle.io.Dataset`): Dataset instance.trans_fn(obj:`callable`, optional, defaults to `None`): function to convert a data sample to input ids, etc.mode(obj:`str`, optional, defaults to obj:`train`): If mode is 'train', it will shuffle the dataset randomly.batch_size(obj:`int`, optional, defaults to 1): The sample number of a mini-batch.batchify_fn(obj:`callable`, optional, defaults to `None`): function to generate mini-batch data by mergingthe sample list, None for only stack each fields of sample in axis0(same as :attr::`np.stack(..., axis=0)`).Returns:dataloader(obj:`paddle.io.DataLoader`): The dataloader which generates batches."""if trans_fn:dataset = dataset.map(trans_fn)shuffle = True if mode == 'train' else Falseif mode == "train":sampler = paddle.io.DistributedBatchSampler(dataset=dataset, batch_size=batch_size, shuffle=shuffle)else:sampler = paddle.io.BatchSampler(dataset=dataset, batch_size=batch_size, shuffle=shuffle)dataloader = paddle.io.DataLoader(dataset, batch_sampler=sampler, collate_fn=batchify_fn)return dataloaderdef convert_example(example, tokenizer, is_test=False):"""Builds model inputs from a sequence for sequence classification tasks. It use `jieba.cut` to tokenize text.Args:example(obj:`list[str]`): List of input data, containing text and label if it have label.tokenizer(obj: paddlenlp.data.JiebaTokenizer): It use jieba to cut the chinese string.is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.Returns:input_ids(obj:`list[int]`): The list of token ids.valid_length(obj:`int`): The input sequence valid length.label(obj:`numpy.array`, data type1 of int64, optional): The input label if not is_test."""input_ids = tokenizer.encode(example["text"])input_ids = np.array(input_ids, dtype='int64')if not is_test:label = np.array(example["label"], dtype="int64")return input_ids, labelelse:return input_ids@paddle.no_grad()
def evaluate(model, criterion, metric, data_loader):"""Given a dataset, it evals model and computes the metric.Args:model(obj:`paddle.nn.Layer`): A model to classify texts.criterion(obj:`paddle.nn.Layer`): It can compute the loss.metric(obj:`paddle.metric.Metric`): The evaluation metric.data_loader(obj:`paddle.io.DataLoader`): The dataset loader which generates batches."""model.eval()metric.reset()losses = []for batch in data_loader:input_ids, token_type_ids, labels = batchlogits = model(input_ids, token_type_ids)loss = criterion(logits, labels)losses.append(loss.numpy())correct = metric.compute(logits, labels)metric.update(correct)accu = metric.accumulate()print("eval loss: %.5f, accu: %.5f" % (np.mean(losses), accu))model.train()metric.reset()

import timefrom utils import evaluate# 训练轮次
epochs = 1
# 训练过程中保存模型参数的文件夹
ckpt_dir = "skep_ckpt"
# len(train_data_loader)一轮训练所需要的step数
num_training_steps = len(train_data_loader) * epochs# Adam优化器
optimizer = paddle.optimizer.AdamW(learning_rate=2e-5,parameters=model.parameters())
# 交叉熵损失函数
criterion = paddle.nn.loss.CrossEntropyLoss()
# accuracy评价指标
metric = paddle.metric.Accuracy()

# 开启训练
global_step = 0
tic_train = time.time()
for epoch in range(1, epochs + 1):for step, batch in enumerate(train_data_loader, start=1):input_ids, token_type_ids, labels = batch# 喂数据给modellogits = model(input_ids, token_type_ids)# 计算损失函数值loss = criterion(logits, labels)# 预测分类概率值probs = F.softmax(logits, axis=1)# 计算acccorrect = metric.compute(probs, labels)metric.update(correct)acc = metric.accumulate()global_step += 1if global_step % 10 == 0:print("global step %d, epoch: %d, batch: %d, loss: %.5f, accu: %.5f, speed: %.2f step/s"% (global_step, epoch, step, loss, acc,10 / (time.time() - tic_train)))tic_train = time.time()# 反向梯度回传，更新参数loss.backward()optimizer.step()optimizer.clear_grad()if global_step % 100 == 0:save_dir = os.path.join(ckpt_dir, "model_%d" % global_step)if not os.path.exists(save_dir):os.makedirs(save_dir)# 评估当前训练的模型evaluate(model, criterion, metric, dev_data_loader)# 保存当前模型参数等model.save_pretrained(save_dir)# 保存tokenizer的词表等tokenizer.save_pretrained(save_dir)

import numpy as np
import paddle# 处理测试集数据
trans_func = partial(convert_example,tokenizer=tokenizer,max_seq_length=max_seq_length,is_test=True)
batchify_fn = lambda samples, fn=Tuple(Pad(axis=0, pad_val=tokenizer.pad_token_id),  # inputPad(axis=0, pad_val=tokenizer.pad_token_type_id),  # segmentStack() # qid
): [data for data in fn(samples)]
test_data_loader = create_dataloader(test_ds,mode='test',batch_size=batch_size,batchify_fn=batchify_fn,trans_fn=trans_func)

# 根据实际运行情况，更换加载的参数路径
params_path = 'skep_ckp/model_500/model_state.pdparams'
if params_path and os.path.isfile(params_path):# 加载模型参数state_dict = paddle.load(params_path)model.set_dict(state_dict)print("Loaded parameters from %s" % params_path)

label_map = {0: '0', 1: '1'}
results = []
# 切换model模型为评估模式，关闭dropout等随机因素
model.eval()
for batch in test_data_loader:input_ids, token_type_ids, qids = batch# 喂数据给模型logits = model(input_ids, token_type_ids)# 预测分类probs = F.softmax(logits, axis=-1)idx = paddle.argmax(probs, axis=1).numpy()idx = idx.tolist()labels = [label_map[i] for i in idx]qids = qids.numpy().tolist()results.extend(zip(qids, labels))

res_dir = "./results"
if not os.path.exists(res_dir):os.makedirs(res_dir)
# 写入预测结果
with open(os.path.join(res_dir, "ChnSentiCorp.tsv"), 'w', encoding="utf8") as f:f.write("index\tprediction\n")for qid, label in results:f.write(str(qid[0])+"\t"+label+"\n")

train_ds, test_ds = load_dataset("seabsa16", "phns", splits=["train", "test"])print(train_ds[0])
print(train_ds[1])
print(train_ds[2])

# 指定模型名称一键加载模型
model = SkepForSequenceClassification.from_pretrained('skep_ernie_1.0_large_ch', num_classes=len(train_ds.label_list))
# 指定模型名称一键加载tokenizer
tokenizer = SkepTokenizer.from_pretrained('skep_ernie_1.0_large_ch')

from functools import partial
import os
import timeimport numpy as np
import paddle
import paddle.nn.functional as F
from paddlenlp.data import Stack, Tuple, Paddef convert_example(example,tokenizer,max_seq_length=512,is_test=False,dataset_name="chnsenticorp"):"""Builds model inputs from a sequence or a pair of sequence for sequence classification tasksby concatenating and adding special tokens. And creates a mask from the two sequences passed to be used in a sequence-pair classification task.A skep_ernie_1.0_large_ch/skep_ernie_2.0_large_en sequence has the following format:::- single sequence: ``[CLS] X [SEP]``- pair of sequences: ``[CLS] A [SEP] B [SEP]``A skep_ernie_1.0_large_ch/skep_ernie_2.0_large_en sequence pair mask has the following format:::0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1| first sequence    | second sequence |If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).note: There is no need token type ids for skep_roberta_large_ch model.Args:example(obj:`list[str]`): List of input data, containing text and label if it have label.tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer` which contains most of the methods. Users should refer to the superclass for more information regarding methods.max_seq_len(obj:`int`): The maximum total input sequence length after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded.is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.dataset_name((obj:`str`, defaults to "chnsenticorp"): The dataset name, "chnsenticorp" or "sst-2".Returns:input_ids(obj:`list[int]`): The list of token ids.token_type_ids(obj: `list[int]`): List of sequence pair mask.label(obj:`numpy.array`, data type of int64, optional): The input label if not is_test."""encoded_inputs = tokenizer(text=example["text"],text_pair=example["text_pair"],max_seq_len=max_seq_length)input_ids = encoded_inputs["input_ids"]token_type_ids = encoded_inputs["token_type_ids"]if not is_test:label = np.array([example["label"]], dtype="int64")return input_ids, token_type_ids, labelelse:return input_ids, token_type_ids

# 处理的最大文本序列长度
max_seq_length=256
# 批量数据大小
batch_size=16# 将数据处理成model可读入的数据格式
trans_func = partial(convert_example,tokenizer=tokenizer,max_seq_length=max_seq_length)
# 将数据组成批量式数据，如
# 将不同长度的文本序列padding到批量式数据中最大长度
# 将每条数据label堆叠在一起
batchify_fn = lambda samples, fn=Tuple(Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input_idsPad(axis=0, pad_val=tokenizer.pad_token_type_id),  # token_type_idsStack(dtype="int64")  # labels
): [data for data in fn(samples)]
train_data_loader = create_dataloader(train_ds,mode='train',batch_size=batch_size,batchify_fn=batchify_fn,trans_fn=trans_func)

# 训练轮次
epochs = 3
# 总共需要训练的step数
num_training_steps = len(train_data_loader) * epochs
# 优化器
optimizer = paddle.optimizer.AdamW(learning_rate=5e-5,parameters=model.parameters())
# 交叉熵损失
criterion = paddle.nn.loss.CrossEntropyLoss()
# Accuracy评价指标
metric = paddle.metric.Accuracy()

# 开启训练
ckpt_dir = "skep_aspect"
global_step = 0
tic_train = time.time()
for epoch in range(1, epochs + 1):for step, batch in enumerate(train_data_loader, start=1):input_ids, token_type_ids, labels = batch# 喂数据给modellogits = model(input_ids, token_type_ids)# 计算损失函数值loss = criterion(logits, labels)# 预测分类概率probs = F.softmax(logits, axis=1)# 计算acccorrect = metric.compute(probs, labels)metric.update(correct)acc = metric.accumulate()global_step += 1if global_step % 10 == 0:print("global step %d, epoch: %d, batch: %d, loss: %.5f, acc: %.5f, speed: %.2f step/s"% (global_step, epoch, step, loss, acc,10 / (time.time() - tic_train)))tic_train = time.time()# 反向梯度回传，更新参数loss.backward()optimizer.step()optimizer.clear_grad()if global_step % 100 == 0:save_dir = os.path.join(ckpt_dir, "model_%d" % global_step)if not os.path.exists(save_dir):os.makedirs(save_dir)# 保存模型参数model.save_pretrained(save_dir)# 保存tokenizer的词表等tokenizer.save_pretrained(save_dir)

@paddle.no_grad()
def predict(model, data_loader, label_map):"""Given a prediction dataset, it gives the prediction results.Args:model(obj:`paddle.nn.Layer`): A model to classify texts.data_loader(obj:`paddle.io.DataLoader`): The dataset loader which generates batches.label_map(obj:`dict`): The label id (key) to label str (value) map."""model.eval()results = []for batch in data_loader:input_ids, token_type_ids = batchlogits = model(input_ids, token_type_ids)probs = F.softmax(logits, axis=1)idx = paddle.argmax(probs, axis=1).numpy()idx = idx.tolist()labels = [label_map[i] for i in idx]results.extend(labels)return results

# 处理测试集数据
label_map = {0: '0', 1: '1'}
trans_func = partial(convert_example,tokenizer=tokenizer,max_seq_length=max_seq_length,is_test=True)
batchify_fn = lambda samples, fn=Tuple(Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input_idsPad(axis=0, pad_val=tokenizer.pad_token_type_id),  # token_type_ids
): [data for data in fn(samples)]
test_data_loader = create_dataloader(test_ds,mode='test',batch_size=batch_size,batchify_fn=batchify_fn,trans_fn=trans_func)

# 根据实际运行情况，更换加载的参数路径
params_path = 'skep_ckpt/model_900/model_state.pdparams'
if params_path and os.path.isfile(params_path):# 加载模型参数state_dict = paddle.load(params_path)model.set_dict(state_dict)print("Loaded parameters from %s" % params_path)results = predict(model, test_data_loader, label_map)

# 写入预测结果
with open(os.path.join("results", "SE-ABSA16_PHNS.tsv"), 'w', encoding="utf8") as f:f.write("index\tprediction\n")for idx, label in enumerate(results):f.write(str(idx)+"\t"+label+"\n")

#将预测文件结果压缩至zip文件，提交
!zip -r results.zip results