在Linux中使用C++调用Python程序

1 0 0

为什么要用C++调Python

我们训练部署CNN模型时，服务器用Pytorch测试的精度比我们部署端精度高0.5%。经过多方排查，发现是由于Pytorch预处理图片使用PIL进行图片读取和尺寸调整，但是部署端采用OpenCV进行图片读取和尺寸调整。两种实现方式实现的Resize操作差异非常大。为了快速完成项目，暂时用C++调用Python的PIL库进行图像预处理，检查实现的精度。

安装Python环境

因为C在编译的时候需要链接到Python函数的一些链接库，并且Linux环境下更改默认Python可能会造成桌面系统损坏，所以我们重新编译了一个Python环境。

1. 安装Python所需的依赖项sudo apt install libbz2-dev

2. 下载Python源代码：https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz

3. 新建python_env目录，并将tgz压缩包复制到其中，使用tar -xvf Python-3.10.14.tgz 解压该文件，得到如下目录结构：

python_env|

| Python-3.10.14

| Python-3.10.14.tgz

4.cd Python-3.10.14

5.配置Python安装文件，打开–enable-shared生成可以连接到g++的动态链接库文件，使用—prefix指定安装目录：

./configure –enable-shared –prefix=/home/dell/Work4/workspace/user_work4/python_env

6.make

7.make -j16 install

8.此时所需的Python环境已经安装到/home/dell/Work4/workspace/user_work4/python_env目录下

9.安装成功后会显示需要添加动态链接库的位置：LD_LIBRARY_PATH

10.将之前显示的动态链接库路径添加到环境变量（可以放到~/.bashrc下）：

export LD_LIBRARY_PATH=/home/dell/Work4/workspace/user_work4/python_env/Python-3.10.14

安装Pytorch OpenCV

在对应的python环境下安装pytorch：

/home/dell/Work4/workspace/user_work4/python_env/bin/python3 -m pip install torch torchvision torchaudio –index-url https://download.pytorch.org/whl/cpu

在对应的python环境下安装opencv：

/home/dell/Work4/workspace/user_work4/python_env/bin/python3 -m pip install opencv-python

编写Python脚本

编写Python脚本pil_load_pic_mod.py

from PIL import Image
import os
import cv2
os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
import numpy as np
import torchvision.transforms as transforms
import pdb

def pil_load_pic(img_path):
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
    trans=transforms.Compose([transforms.Resize(256),transforms.CenterCrop(224),transforms.ToTensor(),normalize])

    # PIL Loading PIC
    # img = Image.open(img_path)
    # img_rgb = img.convert('RGB')

    # Opencv Loading PIC
    img = cv2.imread(img_path)
    img_rgb = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
    img_rgb = Image.fromarray(img_rgb)

    # Resize
    img_resized = trans(img_rgb)
    img_numpy = img_resized.numpy()

    # pdb.set_trace()
    # print("Python Successfully load %s"%img_path) #dtype=float32
    return img_numpy.reshape(-1).tolist()

if __name__=="__main__":
    print(pil_load_pic("/home/dell/Work3/data_set/ImageNet/ILSVRC2012_img_val/ILSVRC2012_val_00000001.JPEG"))

#234
#!/home/dell/Disk03/students/user/python_env/bin/python3
#112
#!/home/dell/Disk03/user/python_env/bin/python3

编写C++程序

#include<stdio.h>
#include<Python.h>

int main(){
    // PIL load and resize pic Init
    float mean[3]={0.485, 0.456, 0.406}, std[3]={0.229, 0.224, 0.225}; //rgb
    #define PIC_OUT_SIZE 224
    Py_Initialize();
    if (!Py_IsInitialized()) {
        printf("ERROR: Python init fail
");
        exit(1);
    }
    PyRun_SimpleString("import sys");
    PyRun_SimpleString("sys.path.append('./')");
    PyObject *pModule = PyImport_ImportModule("pil_load_pic_mod");
    if (pModule == NULL){
        printf("ERROR: No Python module loaded!
");
        PyErr_Print();
        exit(1);
    }

    PyObject *pFunc = PyObject_GetAttrString(pModule, "pil_load_pic");
    if (!PyCallable_Check(pFunc)){
        printf("ERROR: No Python module loaded!
");
        PyErr_Print();
        exit(1);
    }

    char picture_file_path[255] = {0};
    char *img_resized;
    img_resized = (char *)malloc(PIC_OUT_SIZE*PIC_OUT_SIZE*3);
    for (int pic_num = 1; pic_num <=1; pic_num++){

        sprintf(picture_file_path, "/home/dell/Work4/workspace/user_work4/c_and_python/ILSVRC2012_val_%08d.JPEG", pic_num);

        // OpenCV load and resize pic
        // cv_load_pic(picture_file_path,img_resized);

        // PIL load and resize pic
        PyObject *loadImageArgs = Py_BuildValue("(s)", picture_file_path);
        PyObject *pValue = PyObject_CallObject(pFunc, loadImageArgs);
        Py_DECREF(loadImageArgs);

        if (pValue != NULL) {
            printf("Python Successfully load %s
",picture_file_path);
            if (PyList_Check(pValue)) {
                Py_ssize_t size = PyList_Size(pValue);
                if(size != (PIC_OUT_SIZE*PIC_OUT_SIZE*3)){
                    printf("PIL output pic size error!
");
                    exit(1);
                }
                int row=0, col=0, cc=0, out_idx=0;
                for (Py_ssize_t i = 0; i < size; i++) {
                    PyObject *item = PyList_GetItem(pValue, i);

                    // if(col<3 && row<3){
                    //     printf("col:%d row:%d cc:%d %f
", col, row, cc, PyFloat_AsDouble(item));
                    // }
                    img_resized[out_idx++] = (PyFloat_AsDouble(item) * std[cc] + mean[cc]) * 255;

                    if(cc==3-1 && row==PIC_OUT_SIZE-1 && col==PIC_OUT_SIZE-1){
                        cc = 0;
                    }
                    else if(row==PIC_OUT_SIZE-1 && col==PIC_OUT_SIZE-1){
                        cc++;
                    }

                    if(row==PIC_OUT_SIZE-1 && col==PIC_OUT_SIZE-1){
                        row = 0;
                    }
                    else if(col==PIC_OUT_SIZE-1){
                        row++;
                    }

                    if(col==PIC_OUT_SIZE-1){
                        col = 0;
                    }
                    else{
                        col++;
                    }
                }
            }
            Py_DECREF(pValue);
        } else {
            PyErr_Print();
        }
    }
    return 0;
}

编写Makefile

输入指令查看所需编译参数：

/home/dell/Work4/workspace/user_work4/python_env/bin/python3-config –cflags –embed

输入指令查看所需链接参数：

# /home/dell/Work4/workspace/user_work4/python_env/bin/python3-config –ldflags –embed

最终编写Makefile：

# /home/dell/Work4/workspace/user_work4/python_env/bin/python3-config --cflags --embed
COMPILE_FLAG = -I/home/dell/Work4/workspace/user_work4/python_env/include/python3.10 -I/home/dell/Work4/workspace/user_work4/python_env/include/python3.10  -Wno-unused-result -Wsign-compare  -DNDEBUG -g -fwrapv -O3 -Wall

# /home/dell/Work4/workspace/user_work4/python_env/bin/python3-config --ldflags --embed
LINK_FLAG = -L/home/dell/Work4/workspace/user_work4/python_env/lib -lpython3.10 -lcrypt -lpthread -ldl  -lutil -lm -lm

.PHONY: host

host: main_c.o
    gcc main_c.o -o run.exe $(LINK_FLAG)

main_c.o: main_c.cpp
    gcc main_c.cpp -c -o main_c.o $(COMPILE_FLAG)

# 随笔