安装和简单使用Milvus

1 介绍

Milvus是国产的高性能分布式向量数据库。

# Milvus官网
https://milvus.io/# 安装文档
https://milvus.io/docs/install-overview.md# Python的对应关系和接口文档
https://milvus.io/api-reference/pymilvus/v2.4.x/About.md

2 安装Milvus

2.1 安装数据库

# 下载shell脚本
wget https://github.com/milvus-io/milvus/releases/download/v2.4.13/milvus-standalone-docker-compose.yml -O docker-compose.yml# 执行命令
docker-compose -f milvus-standalone-docker-compose.yml up -d# 查看容器
docker ps | grep milvus

修改认证权限

# Milvus的配置目录，一般不用
# 下载milvus.yaml文件
# 下载地址
https://raw.githubusercontent.com/milvus-io/milvus/v2.4.13/configs/milvus.yaml# 修改milvus.yaml文件
...
common:
...security:# 修改milvus.yaml中的下面参数为: trueauthorizationEnabled: false
...# 在milvus-standalone-docker-compose.yml中添加共享数据卷
...# 安装milvusstandalone:container_name: milvus-standalone...volumes:# Milvus的配置目录，将修改后的文件放在下面即可- /home/milvus/milvus/configs/milvus.yaml:/milvus/configs/milvus.yaml...

milvus-standalone-docker-compose.yml

version: '3.5'services:# 安装etcdetcd:container_name: milvus-etcdimage: quay.io/coreos/etcd:v3.5.5restart: alwaysenvironment:- ETCD_AUTO_COMPACTION_MODE=revision- ETCD_AUTO_COMPACTION_RETENTION=1000- ETCD_QUOTA_BACKEND_BYTES=4294967296- ETCD_SNAPSHOT_COUNT=50000volumes:- /home/milvus/etcd:/etcdcommand: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcdhealthcheck:test: ["CMD", "etcdctl", "endpoint", "health"]interval: 30stimeout: 20sretries: 3# 安装miniominio:container_name: milvus-minioimage: minio/minio:RELEASE.2023-03-20T20-16-18Zrestart: alwaysenvironment:MINIO_ACCESS_KEY: minioadminMINIO_SECRET_KEY: minioadminports:- "9001:9001"- "9000:9000"volumes:- /home/milvus/minio:/minio_datacommand: minio server /minio_data --console-address ":9001"healthcheck:test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]interval: 30stimeout: 20sretries: 3# 安装milvusstandalone:container_name: milvus-standaloneimage: milvusdb/milvus:v2.4.13restart: alwayscommand: ["milvus", "run", "standalone"]security_opt:- seccomp:unconfinedenvironment:ETCD_ENDPOINTS: etcd:2379MINIO_ADDRESS: minio:9000volumes:# Milvus的配置目录，可以不配置# 下载地址：https://raw.githubusercontent.com/milvus-io/milvus/v2.4.13/configs/milvus.yaml# common:#  security:#    # 修改milvus.yaml中的下面参数为: true#    authorizationEnabled: false- /home/milvus/milvus/configs/milvus.yaml:/milvus/configs/milvus.yaml# Milvus的数据目录- /home/milvus/milvus/data:/var/lib/milvushealthcheck:test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]interval: 30sstart_period: 90stimeout: 20sretries: 3ports:- "19530:19530"- "9091:9091"depends_on:- "etcd"- "minio"networks:default:name: milvus

2.2 安装工具

⚠️ 注意：attu和Mivus有版本对应关系。

docker run -itd \
--name milvus-attu \
--restart always \
-p 3000:3000 \
zilliz/attu:v2.4.8

访问attu

# 访问地址
http://192.168.108.160:3000/#/# 访问成功后输入Milvus的地址和端口号即可
# Mlivus的默认账号/密码：root/Milvus
192.168.108.160:19530

3 使用Milvus

3.1 安装依赖

# 注意有版本对应关系
pip install pymilvus==2.4.8 -i https://pypi.tuna.tsinghua.edu.cn/simple

3.2 简单使用

import jsonfrom pymilvus import MilvusClient, FieldSchema, DataType
from sentence_transformers import SentenceTransformerdb_name = "test_db"
collection_name = "test_collection"def create_db():# 连接Milvusclient_tmp = MilvusClient(uri="http://192.168.108.160:19530",# 默认的账号和密码token="root:Milvus",# 连接默认数据库db_name="default")# 创建数据库client_tmp.create_database(db_name)def create_collection():# 连接Milvusclient = MilvusClient(uri="http://192.168.108.160:19530",# 默认的账号和密码token="root:Milvus",db_name=db_name# db_name="default")# 创建集合# 判断索引是否存在if client.has_collection(collection_name=collection_name):# 删除集合client.drop_collection(collection_name=collection_name)# 1 设置schemaschema_config = MilvusClient.create_schema(auto_id=False,enable_dynamic_field=True,)# 2 设置索引index_params_config = client.prepare_index_params()# 设置主键schema_config.add_field(field_name="id", datatype=DataType.INT64, is_primary=True)index_params_config.add_index(field_name="id",# 索引设置标量index_type="STL_SORT")# 设置评分类型schema_config.add_field(field_name="score", datatype=DataType.FLOAT)# 设置字符类型schema_config.add_field(field_name="summary", datatype=DataType.VARCHAR, max_length=300)# 设置向量schema_config.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=384)index_params_config.add_index(field_name="vector",index_type="AUTOINDEX",metric_type="COSINE")# 创建索引client.create_collection(collection_name=collection_name,# 设置schemaschema=schema_config,index_params=index_params_config,# 自动编号auto_id=True,# 开启自动动态属性enable_dynamic_field=True)def add_data_vec():# 连接Milvusclient = MilvusClient(uri="http://192.168.108.160:19530",# 默认的账号和密码token="root:Milvus",db_name=db_name)# 生成向量model = SentenceTransformer(model_name_or_path="E:/model/sentencetransformers/all-MiniLM-L6-v2")# 句子列表sentences = ["This framework generates embeddings for each input sentence","Sentences are passed as a list of string.","The quick brown fox jumps over the lazy dog.",]# 注意：all-MiniLM-L6-v2的输出维度是384sentence_embeddings = model.encode(sentences)# 向量列表data_list = list()# 打印嵌入模型i = 0for sentence, embedding in zip(sentences, sentence_embeddings):print("Sentence:", sentence)print("Embedding:", embedding)data_item = dict()data_item["id"] = idata_item["score"] = 0.1data_item["summary"] = sentencedata_item["vector"] = embedding.tolist()print(data_item)data_list.append(data_item)# 增加编号i = i + 1# 设置索引名称res = client.insert(collection_name=collection_name, data=data_list)# 返回值print(res)passdef query_data():# 连接Milvusclient = MilvusClient(uri="http://192.168.108.160:19530",# 默认的账号和密码token="root:Milvus",db_name=db_name)# 生成向量model = SentenceTransformer(model_name_or_path="E:/model/sentencetransformers/all-MiniLM-L6-v2")sentence_embeddings = model.encode("my dog")res = client.search(collection_name=collection_name,# 设置向量# 例子： data = [ [0.3580376395471989, -0.6023495712049978, ……] ]data=[sentence_embeddings.tolist()],# 最大返回值数量limit=5,# 设置搜索参数search_params={"metric_type": "COSINE", "params": {}},# 设置实体中输出的参数output_fields=["score", "summary"])print(res)# 美化输出的缩进量：indent=4result = json.dumps(res, indent=4)print(result)if __name__ == '__main__':# 1 创建数据库# create_db()# 2 创建集合# create_collection()# 3 添加向量# add_data_vec()# 4 查询数据query_data()