git clone https://github.com/ggerganov/llama.cpp cd llama.cpp
🖥️ CPU 版本
cmake -B build_cpu cmake --build build_cpu --config Release
🖥️ CUDA 版本
cmake -B build_cuda -DLLAMA_CUDA=ON cmake --build build_cuda --config Release -j 12
cmake -B build
cmake --build build --config Release -t llama-server
量化
1.将 safetensors 格式转成 gguf
cd ~/code/llama.cpp/build_cuda/bin
python convert-hf-to-gguf.py /mnt/workspace/Qwen2.5-7B-Instruct --outfile /mnt/workspace/Qwen2.5-7B-Instruct-GGUF/Qwen2.5-7B-Instruct-q8_0-v1.gguf --outtype q8_0
2.将 gguf 格式进行(再)量化
cd ~/code/llama.cpp/build_cuda/bin
./quantize --allow-requantize /root/autodl-tmp/models/Llama3-8B-Chinese-Chat-GGUF/Llama3-8B-Chinese-Chat-q8_0-v2_1.gguf /root/autodl-tmp/models/Llama3-8B-Chinese-Chat-GGUF/Llama3-8B-Chinese-Chat-q4_1-v1.gguf Q4_1
部署服务:
cd llama.cpp/build/bin
./llama-server -m /mnt/workspace/Qwen2.5-7B-Instruct-GGUF/Qwen2.5-7B-Instruct-q8_0-v1.gguf/Qwen2.5-7B-Instruct-Q8_0.gguf --port 8080