Reference: I work on machine learning.

Candidate A: I work.

Candidate B: He works on machine learning.

import evaluate

metric = evaluate.load("squad")

metric.compute(predictions=predicted_answers, references=theoretical_answers)

{'exact_match': 83.0, 'f1': 88.25}

Reference: I work on machine learning.

Candidate 1: He works on machine learning.

labels_texts = [["description1 in paper1", "description2 in paper1"], ["description1 in paper2"]]

prediction_strings = ["description1 in paper1. description2 in paper1", "description1 in paper2"]

# 评估模型输出

def evaluate(predictions, references):

    y_true = []

    y_pred = []

    for ref, pred in zip(references, predictions):

        ref_tokens = tokenizer.tokenize(" ".join(ref))

        pred_tokens = tokenizer.tokenize(pred)

        common = set(ref_tokens) & set(pred_tokens)

        y_true.extend([1] * len(common) + [0] * (len(ref_tokens) - len(common)))

        y_pred.extend([1] * len(common) + [0] * (len(pred_tokens) - len(common)))

    precision, recall, f1, _ = precision_recall_fscore_support(y_true, y_pred, average='binary')

    return precision, recall, f1

precision, recall, f1 = evaluate(prediction_strings, labels_texts)

print(f"Precision: {precision:.4f}, Recall: {recall:.4f}, F1 Score: {f1:.4f}")

        # 两个字符串长度不一样, 会报错, 截断比较

        min_len = min(len(ref_tokens), len(pred_tokens))

        ref_tokens = ref_tokens[:min_len]

        pred_tokens = pred_tokens[:min_len]

Prediction Strings: ['impossible. In this paper, we aim to solve this problem by introducing FAR-Trans, the first public dataset for FAR, containing pricing in- formation and retail investor transactions acquired from a large European financial institution.']

Precision: 1.0000, Recall: 0.9750, F1 Score: 0.9873

ROUGE-N = Candidate ∩ Reference l e n ( Reference ) \text{ROUGE-N} = \frac{\text{Candidate} \cap \text{Reference}}{len(\text{Reference})}

ROUGE-L = 最长公共子串 ( Candidate , Reference ) l e n ( Reference ) \text{ROUGE-L} = \frac{\text{最长公共子串}(\text{Candidate}, \text{Reference})}{len(\text{Reference})}

from rouge_score import rouge_scorer

scorer = rouge_scorer.RougeScorer(['rouge1','rouge2', 'rougeL'], use_stemmer=True)

scores = scorer.score('The quick brown fox jumps over the lazy dog',

                      'The quick brown dog jumps on the log.')

print(scores["rouge1"])

print(scores["rouge2"])

print(scores["rougeL"])

#同一个文档拼接到同一个字符串

==================rouge====================

from nltk.translate.bleu_score import sentence_bleu

from nltk.translate.rouge_score import rouge_n, rouge_scorer

def gouge(evaluated_sentences, reference_sentences):

    """

    :param evaluated_sentences: 生成的摘要句子列表

    :param reference_sentences: 参考摘要句子列表

    :return: GOUGE指标

    """

    scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rouge    3', 'rouge4'])

    scores = scorer.score(' '.join(evaluated_sentences),

                          ' '.join(reference_sentences))

    rouge_n_scores = [scores[f'rouge{i}'].precision for i in range(1, 5)]

    return np.exp(np.mean(np.log(rouge_n_scores)))

# 不同文档分别计算评估指标

# importing the native rouge library

from rouge_score import rouge_scorer

# a list of the hypothesis documents

hyp = ['This is the first sample', 'This is another example']

# a list of the references documents

ref = ['This is the first sentence', 'It is one more sentence']

# make a RougeScorer object with rouge_types=['rouge1']

scorer = rouge_scorer.RougeScorer(['rouge1'])

# a dictionary that will contain the results

results = {'precision': [], 'recall': [], 'fmeasure': []}

# for each of the hypothesis and reference documents pair

for (h, r) in zip(hyp, ref):

    # computing the ROUGE

    score = scorer.score(h, r)

    # separating the measurements

    precision, recall, fmeasure = score['rouge1']

    # add them to the proper list in the dictionary

    results['precision'].append(precision)

    results['recall'].append(recall)

    results['fmeasure'].append(fmeasure)

print(results)

{'precision': [0.8, 0.2], 'recall': [0.8, 0.25], 'fmeasure': [0.8000000000000002, 0.22222222222222224]}

def rouge_evaluate(predictions, refs):

    scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)

    rouge_scores = {'rouge1': [], 'rouge2': [], 'rougeL': []}

    for ref, pred in zip(refs, predictions):

        score = scorer.score(" ".join(ref), pred)

        for key in rouge_scores:

            rouge_scores[key].append(score[key].fmeasure)

    avg_rouge_scores = {key: sum(scores) / len(scores) for key, scores in rouge_scores.items()}

    #指标直接储存在文件中

    with open('output/evaluation_rouge.txt', 'w') as eval_file:

        # eval_file.write(f"rouge1: {precision:.4f}\n")

        # eval_file.write(f"rouge2: {recall:.4f}\n")

        # eval_file.write(f"rougeL: {f1:.4f}\n")

        eval_file.write(f"ROUGE Scores:\n")

        for key, score in avg_rouge_scores.items():

            eval_file.write(f"{key}: {score:.4f}\n")

    return avg_rouge_scores

rouge_results = rouge_evaluate(dataset_descriptions, labels_texts)

from sklearn.model_selection import train_test_split

# Split the dataset into training and evaluation sets

train_size = 0.8

train_indices, val_indices = train_test_split(list(range(len(inputs["input_ids"]))), train_size=train_size, random_state=42)

train_inputs = {key: val[train_indices] for key, val in inputs.items()}

val_inputs = {key: val[val_indices] for key, val in inputs.items()}

train_dataset = TensorDataset(train_inputs["input_ids"], train_inputs["attention_mask"], train_inputs["labels"])

val_dataset = TensorDataset(val_inputs["input_ids"], val_inputs["attention_mask"], val_inputs["labels"])

train_dataloader = DataLoader(train_dataset, batch_size=2, shuffle=True)

val_dataloader = DataLoader(val_dataset, batch_size=2, shuffle=False)

#训练时

for batch in train_dataloader:

…

avg_epoch_loss = epoch_loss / len(train_dataloader)

#评估时

# Evaluate on the validation set

val_predictions, val_labels = evaluate(model, val_dataloader, tokenizer)

…

precision, recall, f1, rouge_scores = calculate_metrics(val_predictions, val_labels, tokenizer)

val_dataset = TensorDataset(val_inputs["input_ids"], val_inputs["attention_mask"], val_inputs["labels"], val_inputs["paper_idx"])

# Split the original data into training and evaluation subsets

train_sentences, val_sentences, train_labels_texts, val_labels_texts, train_titles, val_titles = train_test_split(

    sentences, labels_texts, titles, train_size=0.8, random_state=42

)

# Tokenize the training and validation data separately

train_inputs = tokenize_and_align_labels(train_sentences, train_labels_texts, train_titles, tokenizer)

val_inputs = tokenize_and_align_labels(val_sentences, val_labels_texts, val_titles, tokenizer)

model.to(device)

train_inputs = {key: val.to(device) for key, val in train_inputs.items()}

val_inputs = {key: val.to(device) for key, val in val_inputs.items()}

train_dataset = TensorDataset(train_inputs["input_ids"], train_inputs["attention_mask"], train_inputs["labels"])

val_dataset = TensorDataset(val_inputs["input_ids"], val_inputs["attention_mask"], val_inputs["labels"])

train_dataloader = DataLoader(train_dataset, batch_size=2, shuffle=True)

val_dataloader = DataLoader(val_dataset, batch_size=2, shuffle=False)#防止生成的同一篇文章断开

precision, recall, f1 = token_evaluate(dataset_descriptions, val_labels_texts)

rouge_results = rouge_evaluate(dataset_descriptions, val_labels_texts)

dataset_descriptions = get_extracted_description(val_predictions, val_inputs["input_ids"], val_inputs["paper_idx"])

    tokenized_inputs = {"input_ids": [], "attention_mask": [], "labels": [], "paper_idx": [], "token_positions": []}

for j in range(len(input_ids)):#同一篇文章中的每一个滑窗

            tokenized_sentence = tokenizer.convert_ids_to_tokens(input_ids[j])

            token_label = [0] * len(tokenized_sentence)

            main_body_start=find_main_body(tokenized_sentence)

            token_positions = []

            for label_text in labels_text:

                # print("label_text:",label_text)

                tokenized_label = tokenizer.tokenize(label_text)

                # print("label_text:",label_text)

                tokenized_label = [token for token in tokenized_label if token != '<pad>']#删除pad

                label_length = len(tokenized_label)

                # print("tokenized_label:",tokenized_label)

                # 处理label跨滑窗

                for k in range(main_body_start,len(tokenized_sentence)-1):#从正文部分开始匹配

                    # print("tokenized_sentence:",tokenized_sentence[k:k + label_length])

                    end_position=min(len(tokenized_sentence)-1, k + label_length)

                    if tokenized_sentence[k:end_position] == tokenized_label[0:end_position]:#后半部分没有的情况

                        print("matched tokenized_label:", tokenized_sentence[k:end_position])

                        # print("matched tokenized_label:",tokenized_label,"\n",tokenized_sentence[k:end_position])

                        token_label[k:end_position] = [1] * (end_position-k)

                        # print("matched clsfy_label:",token_label[k:len(tokenized_sentence)-1])

                        for pos in range(k, end_position):

                            token_positions.append(pos)

                for label_start in range(label_length-1):#前半部分没有的情况

                    # print("tokenized_sentence:",tokenized_sentence[k:k + label_length])

                    if tokenized_sentence[main_body_start:label_length-label_start] == tokenized_label[label_start:]:#后半部分没有的情况

                        print("matched tokenized_label:",tokenized_sentence[main_body_start:label_length-label_start])

                        # print("matched tokenized_label:",tokenized_sentence[main_body_start:label_length-label_start],"\n",tokenized_label[label_start:])

                        token_label[main_body_start:label_length-label_start] = [1] * (label_length-label_start)

                        # print("matched clsfy_label:",token_label[k:len(tokenized_sentence)-1])

                        for pos in range(main_body_start, label_length - label_start):

                            token_positions.append(pos)

            tokenized_inputs["token_positions"].append(token_positions)

pre_token_positions = []

    for idx, (token, pred_class) in enumerate(zip(tokenized_sub_sentence, predicted_tokens_classes)):

        is_descrp=(pred_class == 'Dataset description')

        if(is_descrp):

            pre_token_positions.append(idx)

    return dataset_description_string, pre_token_positions

        dataset_description, pre_token_positions=get_prediction_string(prediction_class, predicted_input_id, is_same_paper)

    papers_pre_token_positions=[]

        dataset_description, pre_token_positions=get_prediction_string(prediction_class, predicted_input_id, is_same_paper)

        papers_pre_token_positions.append(pre_token_positions)

    return dataset_descriptions, papers_pre_token_positions

dataset_descriptions,papers_pre_token_positions = get_extracted_description(val_predictions, val_inputs["input_ids"], val_inputs["paper_idx"])

def token_evaluate(pre_token_positions, label_token_positions):

    true_positive = 0

    false_positive = 0

    false_negative = 0

    for pred_positions, label_positions in zip(pre_token_positions, label_token_positions):

        pred_positions_set = set(pred_positions)

        label_positions_set = set(label_positions)

        true_positive += len(pred_positions_set & label_positions_set)

        false_positive += len(pred_positions_set - label_positions_set)

        false_negative += len(label_positions_set - pred_positions_set)

    precision = true_positive / (true_positive + false_positive) if (true_positive + false_positive) > 0 else 0

    recall = true_positive / (true_positive + false_negative) if (true_positive + false_negative) > 0 else 0

    f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0

    return precision, recall, f1

# 例子

pre_token_positions = [[], [2], []]

label_token_positions = [[1], [2], [3]]

precision, recall, f1 = token_evaluate(pre_token_positions, label_token_positions)

print(f"Precision: {precision}")

print(f"Recall: {recall}")

print(f"F1-score: {f1}")

pip install wandb

wandb login

import wandb

import random

# start a new wandb run to track this script

wandb.init(

    # set the wandb project where this run will be logged

    project="my-awesome-project",

    # track hyperparameters and run metadata

    config={

    "learning_rate": 0.02,

    "architecture": "CNN",

    "dataset": "CIFAR-100",

    "epochs": 10,

    }

)

# simulate training

epochs = 10

offset = random.random() / 5

for epoch in range(2, epochs):

    acc = 1 - 2 ** -epoch - random.random() / epoch - offset

    loss = 2 ** -epoch + random.random() / epoch + offset

    # log metrics to wandb

    wandb.log({"acc": acc, "loss": loss})

# [optional] finish the wandb run, necessary in notebooks

wandb.finish()

wandb: Run data is saved locally in D:\Projects\longformer\wandb\run-20240805_204153-4hznct2i

wandb: Run `wandb offline` to turn off syncing.

wandb: Syncing run hearty-eon-1

wandb:  View project at https://wandb.ai/lalagoon-north-china-electric-power-university/test-project

wandb:  View run at https://wandb.ai/lalagoon-north-china-electric-power-university/test-project/runs/4hznct2i

import wandb

# start a new wandb run to track this script

wandb.init(

    # set the wandb project where this run will be logged

    project="extract-dataset-description-project",

    # track hyperparameters and run metadata

    config={

    "learning_rate": 5e-5,

    "architecture": "Longformer",

    "dataset": "Description-500",

    "epochs": 100,

    }

)

# 记录损失到 wandb

wandb.log({"loss": avg_epoch_loss})

wandb.finish()

export WANDB_API_KEY='xxxx'

            if(filter_empty_window and len(token_positions)==0):

                continue

train_inputs = tokenize_and_align_labels(train_papers, tokenizer,filter_empty_window=True)

descri_file_path='input/papers_and_datasets.json'

def read_json(file_path):

    with open(file_path, 'r', encoding='utf-8') as file:

        data = json.load(file)

    return data

papers_info = read_json(descri_file_path)

sentences = data['paper_texts']

labels_texts = data['dataset_descriptions']

titles= data['titles']

train_papers, val_papers = train_test_split(

    papers_info, train_size=0.8, random_state=42

)

train_inputs = tokenize_and_align_labels(train_papers, tokenizer)

val_inputs = tokenize_and_align_labels(val_papers, tokenizer)

def tokenize_and_align_labels(papers, tokenizer, max_length=4096, stride=256):

    for i, paper in enumerate(papers_info):

        sentence=paper.get("paper_text")

        labels_text = paper.get("dataset_descriptions")

        title = paper.get("title")

def token_evaluate(predictions, val_papers, tokenizer):

    for paper, pred in zip(val_papers, predictions):#ref_tokens中是一篇文章的label, y_true.extend后是所有样本的输出

        ref=paper.get("dataset_descriptions")

precision, recall, f1 = token_evaluate(dataset_descriptions, val_papers, tokenizer)

def rouge_evaluate(predictions, val_papers):

    for paper, pred in zip(val_papers, predictions):

        ref=paper.get("dataset_descriptions")

rouge_results = rouge_evaluate(dataset_descriptions, val_papers)

descri_file_path='input/papers_and_datasets.json'

loss_log_path='output/training_loss.txt'

loss_fig_path='output/training_loss.png'

model_save_path="output/trained_model"

eval_token_path='output/evaluation_token.txt'

eval_rouge_path='output/evaluation_rouge.txt'

input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)

        # OutOfMemoryError: CUDA out of memory. Tried to allocate 24.00 MiB. GPU

RuntimeError: NPU out of memory. Tried to allocate 578.00 MiB (NPU 0; 60.97 GiB total capacity; 8.23 GiB already allocated; 8.23 GiB current active; 362.04 MiB free; 8.95 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.

import torch

torch.cuda.empty_cache()

torch_npu.npu.empty_cache()

export 'PYTORCH_NPU_ALLOC_CONF=max_split_size_mb:512'

torch._C._debug_set_max_split_size_mb(512)

torch_npu._C._debug_set_max_split_size_mb(512)

npu-smi info

+---------------------------+---------------+----------------------------------------------------+

| NPU     Chip              | Process id    | Process name             | Process memory(MB)      |

+===========================+===============+====================================================+

| 0       0                 | 1349884       | python                   | 30216                   |

+===========================+===============+====================================================+

os.environ["NPU_VISIBLE_DEVICES"] = "1,2,3"

device = torch.device("npu:1" if torch.npu.is_available() else "npu:2")

def select_available_npu(required_memory_mb):

    if torch.npu.is_available():

        for i in range(1, 8):

            props = torch.npu.get_device_properties(f"npu:{i}")

            if props.total_memory - props.reserved_memory >= required_memory_mb * 1024 * 1024:

                return torch.device(f"npu:{i}")

    return torch.device("cpu")

查看总内存

torch_npu.npu.get_device_properties("npu:1").total_memory

设置特定npu, 注意这里查询要用torch_npu.npu, 不然取不到

device = torch.device("npu:1" if torch_npu.npu.is_available() else "npu:2")

----+

| NPU     Chip              | Process id    | Process name             | Process memory(MB)      |

+===========================+===============+====================================================+

| 0       0                 | 47784         | python                   | 52657                   |

+===========================+===============+================================================

nohup ping www.baidu.com &

ls

cat nohup.out

ps -ef | grep ping

kill [$PID]

[1] 1255914

nohup: ignoring input and appending output to 'nohup.out'

nohup python tkn_clsfy.py &

ps -ef | grep python

[1] 27308

root       27308   27285 99 13:19 pts/14   00:01:49 python tkn_clsfy.py