1.数据格式转换

模版格式：

[{"id": "identity_0","conversations": [{"from": "user","value": "你好"},{"from": "assistant","value": "我是Qwen-VL,一个支持视觉输入的大模型。"}]},{"id": "identity_1","conversations": [{"from": "user","value": "Picture 1: <img>https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg</img>\n图中的狗是什么品种？"},{"from": "assistant","value": "图中是一只拉布拉多犬。"},{"from": "user","value": "框出图中的格子衬衫"},{"from": "assistant","value": "<ref>格子衬衫</ref><box>(588,499),(725,789)</box>"}]},{ "id": "identity_2","conversations": [{"from": "user","value": "Picture 1: <img>assets/mm_tutorial/Chongqing.jpeg</img>\nPicture 2: <img>assets/mm_tutorial/Beijing.jpeg</img>\n图中都是哪"},{"from": "assistant","value": "第一张图片是重庆的城市天际线，第二张图片是北京的天际线。"}]}
]

import json# 打开并加载.json文件
with open(r"E:\comprehensive_library\e_commerce_lmm\data\openi-zh-prompt.json", 'r', encoding='utf-8') as f:data = json.load(f)# 数据格式转换
new_data = []
for i, d in enumerate(data):new_conversations = dict()new_conversations["id"] = f"identity_{i}"new_conversations["conversations"] = [{"from": "user","value": "Picture 1: <img>" + d["img"] + "</img>\n" + d["prompt"]},{"from": "assistant","value": d["label"]}]new_data.append(new_conversations)# 将新的数据写入新的.json文件中
with open("../data/openai-zh-qwenvl-prompt.json", 'w', encoding='utf-8') as f:json.dump(new_data, f, ensure_ascii=False, indent=4)

2.微调

直接pip install -r requirments.txt

注意gcc要升级到9.3

yum -y install centos-release-scl
yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-binutils
scl enable devtoolset-9 bashecho "source /opt/rh/devtoolset-9/enable" >>/etc/profile

lora在V100上显存不够，微调不起来，在a800上可以。用swift库可以在V100上微调Qwen-vl。

#!/bin/bash
export CUDA_DEVICE_MAX_CONNECTIONS=1
DIR=$(pwd)GPUS_PER_NODE=$(python -c 'import torch; print(torch.cuda.device_count())')
NNODES=1
NODE_RANK=0
MASTER_ADDR=localhost
MASTER_PORT=6001MODEL="/home/image_team/image_team_docker_home/lgd/e_commerce_lmm/weights/qwen-vl-caht/" #"Qwen/Qwen-VL-Chat"/"Qwen/Qwen-VL"  Set the path if you do not want to load from huggingface directly
# ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations.
# See the section for finetuning in README for more information.
DATA="/home/image_team/image_team_docker_home/lgd/e_commerce_lmm/data/openai-zh-qwenvl-prompt.json"DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE \--nnodes $NNODES \--node_rank $NODE_RANK \--master_addr $MASTER_ADDR \--master_port $MASTER_PORT
"torchrun $DISTRIBUTED_ARGS finetune.py \--model_name_or_path $MODEL \--data_path $DATA \--bf16 False \--fp16 True \--fix_vit True \--output_dir output_qwen \--num_train_epochs 5 \--per_device_train_batch_size 1 \--per_device_eval_batch_size 1 \--gradient_accumulation_steps 1 \--evaluation_strategy "no" \--save_strategy "steps" \--save_steps 1000 \--save_total_limit 10 \--learning_rate 1e-5 \--weight_decay 0.1 \--adam_beta2 0.95 \--warmup_ratio 0.01 \--lr_scheduler_type "cosine" \--logging_steps 1 \--report_to "none" \--model_max_length 1024 \--lazy_preprocess True \--use_lora \--gradient_checkpointing \--deepspeed finetune/ds_config_zero2.json