milvus 环境搭建

1.下载 milvus-standalone-docker-compose.yml 配置文件并保存为 docker-compose.yml

wget https://github.com/milvus-io/milvus/releases/download/v2.0.2/milvus-standalone-docker-compose.yml -O docker-compose.yml

or

wget https://assets.artlib.cn/assets/soft/milvus-standalone-docker-compose.yml -O docker-compose.yml

2.安装docker

curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun

or

curl -sSL https://get.daocloud.io/docker | sh

3.安装docker-compose

wget https://assets.artlib.cn/assets/soft/docker-composev2.4.1 -O /usr/local/bin/docker-compose

sudo chmod +x /usr/local/bin/docker-compose

sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose

systemctl start docker

4.启动

sudo docker-compose up -d

Attu 搭建

docker run -p 8000:3000 -e HOST_URL=http://{ your machine IP }:8000 -e MILVUS_URL={your machine IP}:19530 zilliz/attu:latest

应用

1.安装anaconda

wget https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-x86_64.sh

or

wget https://assets.artlib.cn/assets/soft/Anaconda3-2022.05-Linux-x86_64.sh

加入环境变量

vim /etc/profile
export PATH=$PATH:/root/anaconda3/bin
source /etc/profile

# 创建环境
conda create --name py3 python=3.6

# 进入环境
conda activate py3
# source activate py3

# 退出环境
conda deactivate

2.安装python包

cat requirements.txt

absl-py==1.1.0
aiohttp==3.8.1
aiosignal==1.2.0
antlr4-python3-runtime==4.9.3
anyio==3.6.1
argcomplete==1.12.3
asgiref==3.4.1
async-timeout==4.0.2
asynctest==0.13.0
attrs==21.4.0
backports.weakref==1.0rc1
black==22.3.0
bleach==1.5.0
boto3==1.23.10
botocore==1.26.10
cached-property==1.5.2
cachetools==4.2.4
certifi==2021.5.30
charset-normalizer==2.0.12
click==8.0.4
colorlog==6.6.0
contextlib2==21.6.0
contextvars==2.4
cycler==0.11.0
dataclasses==0.8
distlib==0.3.4
fastapi==0.78.0
filelock==3.4.1
flake8==4.0.1
flake8-copyright==0.2.2
Flask==2.0.3
flatbuffers==2.0
frozenlist==1.2.0
fsspec==2022.1.0
future==0.18.2
google-auth==2.7.0
google-auth-oauthlib==0.4.6
grpcio==1.37.1
grpcio-tools==1.37.1
h11==0.13.0
h5py==3.1.0
html5lib==0.9999999
httptools==0.3.0
hydra-core==1.2.0
idna==3.3
idna-ssl==1.1.0
immutables==0.18
importlib-metadata==4.2.0
importlib-resources==5.4.0
isort==5.10.1
itsdangerous==2.0.1
Jinja2==3.0.3
jmespath==0.10.0
kiwisolver==1.3.1
Markdown==3.3.4
MarkupSafe==2.0.1
matplotlib==3.3.4
mccabe==0.6.1
mmh3==3.0.0
multidict==5.2.0
mypy-extensions==0.4.3
nox==2022.1.7
numpy==1.19.5
oauthlib==3.2.0
omegaconf==2.2.2
onnx==1.11.0
onnxruntime==1.10.0
opencv-python==4.6.0.66
packaging==21.3
pandas==1.1.5
pathspec==0.9.0
Pillow==8.4.0
platformdirs==2.4.0
protobuf==3.19.4
py==1.11.0
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycodestyle==2.8.0
pydantic==1.9.1
pyDeprecate==0.3.1
pyflakes==2.4.0
pymilvus==2.0.2
pymilvus-orm==2.0.0rc1
pymilvusdm==2.0
PyMySQL==1.0.2
pyparsing==3.0.9
python-dateutil==2.8.2
python-dotenv==0.20.0
python-multipart==0.0.5
pytorch-lightning==1.5.10
pytouch==0.4.0
pytz==2022.1
PyYAML==6.0
requests==2.27.1
requests-oauthlib==1.3.1
rsa==4.8
ruamel.yaml==0.16.6
ruamel.yaml.clib==0.2.6
s3transfer==0.5.2
scipy==1.5.4
six==1.16.0
sniffio==1.2.0
starlette==0.19.1
tabulate==0.8.9
tensorboard==2.9.1
tensorboard-data-server==0.6.1
tensorboard-plugin-wit==1.8.1
tensorflow==1.2.1
timm==0.5.4
tomli==1.2.3
torch==1.10.1
torchmetrics==0.8.2
torchvision==0.11.2
towhee==0.6.1
towhee.models==0.6.0
tqdm==4.64.0
typed-ast==1.5.4
typing_extensions==4.1.1
ujson==4.3.0
urllib3==1.26.9
uuid==1.30
uvicorn==0.16.0
uvloop==0.14.0
virtualenv==20.14.1
watchgod==0.7
websockets==9.1
Werkzeug==2.0.3
yarl==1.7.2
zipp==3.6.0

3.图片特征提取 resnet50

import torch
import torch.nn as nn
from torchvision import models, transforms
from torch.autograd import Variable
from PIL import Image


# 提取图片特征
def get_resnet50_feature(img_path):
    transform1 = transforms.Compose([  # 串联多个图片变换的操作
        # transforms.Scale((256, 256)),  # 缩放
        transforms.Resize((256, 256)),
        transforms.CenterCrop((224, 224)),  # 中心裁剪
        transforms.ToTensor()]  # 转换成Tensor
    )

    img = Image.open(img_path)  # 打开图片
    img1 = transform1(img)  # 对图片进行transform1的各种操作

    resnet50_feature_extractor = models.resnet50(pretrained=True)  # 导入ResNet50的预训练模型
    resnet50_feature_extractor.fc = nn.Linear(2048, 2048)  # 重新定义最后一层
    # torch.nn.init.eye(resnet50_feature_extractor.fc.weight)  # 将二维tensor初始化为单位矩阵
    torch.nn.init.eye_(resnet50_feature_extractor.fc.weight)

    for param in resnet50_feature_extractor.parameters():
        param.requires_grad = False

    x = Variable(torch.unsqueeze(img1, dim=0).float(), requires_grad=False)
    y = resnet50_feature_extractor(x)
    y = y.data.numpy()

    return y

4.导入到milvus

def insert_data(img_path, work_id):
    y = get_resnet50_feature(img_path)

    fields = [
        FieldSchema(name="id", dtype=DataType.INT64, is_primary=True, auto_id=True),
        FieldSchema(name="d1", dtype=DataType.FLOAT_VECTOR, dim=2048)
    ]

    schema = CollectionSchema(fields)

    print("Create collection `hello_milvus`")
    hello_milvus = Collection("h2", schema, consistency_level="Strong")

    insert_result = hello_milvus.insert([y])
    print("insert_result:", insert_result)
    print("primary_keys:", insert_result.primary_keys[0])
    #
    id = insert_result.primary_keys[0]
    insert_to_mysql(mid=id, path=img_path, work_id=work_id)