TensorFlow的推理脚本示例 TensorFlow MnistService示例如下。更多TensorFlow推理代码示例请参考TensorFlow、TensorFlow 2.1。其他引擎推理代码请参考PyTorch、Caffe。 推理代码 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 from PIL import Image import numpy as np from model_service.tfserving_model_service import TfServingBaseService class MnistService(TfServingBaseService): def _preprocess(self, data): preprocessed_data = {} for k, v in data.items(): for file_name, file_content in v.items(): image1 = Image.open(file_content) image1 = np.array(image1, dtype=np.float32) image1.resize((1, 784)) preprocessed_data[k] = image1 return preprocessed_data def _postprocess(self, data): infer_output = {} for output_name, result in data.items(): infer_output["mnist_result"] = result[0].index(max(result[0])) return infer_output 请求 curl -X POST \ 在线服务地址 \ -F images=@test.jpg 返回 {"mnist_result": 7} 在上面的代码示例中，完成了将用户表单输入的图片的大小调整，转换为可以适配模型输入的shape。首先通过Pillow库读取“32×32”的图片，调整图片大小为“1×784”以匹配模型输入。在后续处理中，转换模型输出为列表，用于Restful接口输出展示。

XGBoost的推理脚本示例 更多机器学习引擎的推理代码请参考Pyspark、Scikit Learn。 # coding:utf-8 import collections import json import xgboost as xgb from model_service.python_model_service import XgSklServingBaseService class UserService(XgSklServingBaseService): # request data preprocess def _preprocess(self, data): list_data = [] json_data = json.loads(data, object_pairs_hook=collections.OrderedDict) for element in json_data["data"]["req_data"]: array = [] for each in element: array.append(element[each]) list_data.append(array) return list_data # predict def _inference(self, data): xg_model = xgb.Booster(model_file=self.model_path) pre_data = xgb.DMatrix(data) pre_result = xg_model.predict(pre_data) pre_result = pre_result.tolist() return pre_result # predict result process def _postprocess(self, data): resp_data = [] for element in data: resp_data.append({"predict_result": element}) return resp_data

MindSpore的推理脚本示例 snt3芯片目前只有北京四提工单申请权限后才可以使用，支持模型格式为.om，推理脚本如下： from __future__ import absolute_import from __future__ import division from __future__ import print_function import json import os import numpy as np from PIL import Image from hiai.nn_tensor_lib import NNTensor from hiai.nntensor_list import NNTensorList from model_service.hiai_model_service import HiaiBaseService class DemoService(HiaiBaseService): def __init__(self, *args, **kwargs): # 默认加载模型包目录下的om文件 super(DemoService, self).__init__(*args, **kwargs) self.labels_list = None self.is_multilabel = False def _preprocess(self, data): preprocessed_data = {} images = [] for k, v in data.items(): for file_name, file_content in v.items(): image = Image.open(file_content) image = np.array(image) # NHWC # AIPP should use RGB format. # mean reg is applied in AIPP. # Transpose is applied in AIPP tensor = NNTensor(image) images.append(tensor) tensor_list = NNTensorList(images) preprocessed_data['images'] = tensor_list return preprocessed_data def _inference(self, data, image_info=None): result = {} for k, v in data.items(): result[k] = self.model.proc(v) return result def _postprocess(self, data): # 这里增加自己的后处理 return str(data)

自定义推理逻辑的推理脚本示例 首先，需要在配置文件中，定义自己的依赖包，详细示例请参见使用自定义依赖包的模型配置文件示例。然后通过如下示例代码，实现了“saved_model”格式模型的加载推理。 当前推理基础镜像使用的python的logging模块，采用的是默认的日志级别Warnning，即当前只有warning级别的日志可以默认查询出来。如果想要指定INFO等级的日志能够查询出来，需要在代码中指定logging的输出日志等级为INFO级别。

使用自定义依赖包的模型配置文件示例 如下示例中，定义了1.16.4版本的numpy的依赖环境。 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 { "model_algorithm": "image_classification", "model_type": "TensorFlow", "runtime": "python3.6", "apis": [ { "url": "/", "method": "post", "request": { "Content-type": "multipart/form-data", "data": { "type": "object", "properties": { "images": { "type": "file" } } } }, "response": { "Content-type": "applicaton/json", "data": { "type": "object", "properties": { "mnist_result": { "type": "array", "item": [ { "type": "string" } ] } } } } } ], "metrics": { "f1": 0.124555, "recall": 0.171875, "precision": 0.00234938928519385, "accuracy": 0.00746268656716417 }, "dependencies": [ { "installer": "pip", "packages": [ { "restraint": "EXACT", "package_version": "1.16.4", "package_name": "numpy" } ] } ] }

机器学习类型的模型配置文件示例 以下代码以XGBoost为例。 模型输入： { "req_data": [ { "sepal_length": 5, "sepal_width": 3.3, "petal_length": 1.4, "petal_width": 0.2 }, { "sepal_length": 5, "sepal_width": 2, "petal_length": 3.5, "petal_width": 1 }, { "sepal_length": 6, "sepal_width": 2.2, "petal_length": 5, "petal_width": 1.5 } ] } 模型输出： { "resp_data": [ { "predict_result": "Iris-setosa" }, { "predict_result": "Iris-versicolor" } ] } 配置文件： { "model_type": "XGBoost", "model_algorithm": "xgboost_iris_test", "runtime": "python2.7", "metrics": { "f1": 0.345294, "accuracy": 0.462963, "precision": 0.338977, "recall": 0.351852 }, "apis": [ { "url": "/", "method": "post", "request": { "Content-type": "application/json", "data": { "type": "object", "properties": { "req_data": { "items": [ { "type": "object", "properties": {} } ], "type": "array" } } } }, "response": { "Content-type": "applicaton/json", "data": { "type": "object", "properties": { "resp_data": { "type": "array", "items": [ { "type": "object", "properties": { "predict_result": {} } } ] } } } } } ] }

图像分类模型配置文件示例 如下代码以TensorFlow引擎为例，您可以根据实际使用的引擎类型修改model_type参数后使用。 模型输入 key：images value：图片文件 模型输出 1 2 3 4 5 6 7 { "predicted_label": "flower", "scores": [ ["rose", 0.99], ["begonia", 0.01] ] } 配置文件 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 { "model_type": "TensorFlow", "model_algorithm": "image_classification", "metrics": { "f1": 0.345294, "accuracy": 0.462963, "precision": 0.338977, "recall": 0.351852 }, "apis": [{ "url": "/", "method": "post", "request": { "Content-type": "multipart/form-data", "data": { "type": "object", "properties": { "images": { "type": "file" } } } }, "response": { "Content-type": "application/json", "data": { "type": "object", "properties": { "predicted_label": { "type": "string" }, "scores": { "type": "array", "items": [{ "type": "array", "minItems": 2, "maxItems": 2, "items": [ { "type": "string" }, { "type": "number" } ] }] } } } } }], "dependencies": [{ "installer": "pip", "packages": [{ "restraint": "ATLEAST", "package_version": "1.15.0", "package_name": "numpy" }, { "restraint": "", "package_version": "", "package_name": "Pillow" } ] }] } 如下代码以MindSpore引擎为例，您可以根据实际使用的引擎类型修改model_type参数后使用。 模型输入 key：images value：图片文件 模型输出 1 "[[-2.404526 -3.0476532 -1.9888215 0.45013925 -1.7018927 0.40332815\n -7.1861157 11.290332 -1.5861531 5.7887416 ]]" 配置文件 { "model_algorithm": "image_classification", "model_type": "MindSpore", "metrics": { "f1": 0.124555, "recall": 0.171875, "precision": 0.0023493892851938493, "accuracy": 0.00746268656716417 }, "apis": [{ "url": "/", "method": "post", "request": { "Content-type": "multipart/form-data", "data": { "type": "object", "properties": { "images": { "type": "file" } } } }, "response": { "Content-type": "applicaton/json", "data": { "type": "object", "properties": { "mnist_result": { "type": "array", "item": [{ "type": "string" }] } } } } } ], "dependencies": [] }

目标检测模型配置文件示例 如下代码以TensorFlow引擎为例，您可以根据实际使用的引擎类型修改model_type参数后使用。 模型输入 key：images value：图片文件 模型输出 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 { "detection_classes": [ "face", "arm" ], "detection_boxes": [ [ 33.6, 42.6, 104.5, 203.4 ], [ 103.1, 92.8, 765.6, 945.7 ] ], "detection_scores": [0.99, 0.73] } 配置文件 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 { "model_type": "TensorFlow", "model_algorithm": "object_detection", "metrics": { "f1": 0.345294, "accuracy": 0.462963, "precision": 0.338977, "recall": 0.351852 }, "apis": [{ "url": "/", "method": "post", "request": { "Content-type": "multipart/form-data", "data": { "type": "object", "properties": { "images": { "type": "file" } } } }, "response": { "Content-type": "application/json", "data": { "type": "object", "properties": { "detection_classes": { "type": "array", "items": [{ "type": "string" }] }, "detection_boxes": { "type": "array", "items": [{ "type": "array", "minItems": 4, "maxItems": 4, "items": [{ "type": "number" }] }] }, "detection_scores": { "type": "array", "items": [{ "type": "number" }] } } } } }], "dependencies": [{ "installer": "pip", "packages": [{ "restraint": "EXACT", "package_version": "1.15.0", "package_name": "numpy" }, { "restraint": "EXACT", "package_version": "5.2.0", "package_name": "Pillow" } ] }] }

预测分析模型配置文件示例 如下代码以TensorFlow引擎为例，您可以根据实际使用的引擎类型修改model_type参数后使用。 模型输入 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 { "data": { "req_data": [ { "buying_price": "high", "maint_price": "high", "doors": "2", "persons": "2", "lug_boot": "small", "safety": "low", "acceptability": "acc" }, { "buying_price": "high", "maint_price": "high", "doors": "2", "persons": "2", "lug_boot": "small", "safety": "low", "acceptability": "acc" } ] } } 模型输出 1 2 3 4 5 6 7 8 9 10 11 12 { "data": { "resp_data": [ { "predict_result": "unacc" }, { "predict_result": "unacc" } ] } } 配置文件 代码中request结构和response结构中的data参数是json schema数据结构。data/properties里面的内容对应“模型输入”和“模型输出”。 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 { "model_type": "TensorFlow", "model_algorithm": "predict_analysis", "metrics": { "f1": 0.345294, "accuracy": 0.462963, "precision": 0.338977, "recall": 0.351852 }, "apis": [ { "url": "/", "method": "post", "request": { "Content-type": "application/json", "data": { "type": "object", "properties": { "data": { "type": "object", "properties": { "req_data": { "items": [ { "type": "object", "properties": {} } ], "type": "array" } } } } } }, "response": { "Content-type": "application/json", "data": { "type": "object", "properties": { "data": { "type": "object", "properties": { "resp_data": { "type": "array", "items": [ { "type": "object", "properties": {} } ] } } } } } } } ], "dependencies": [ { "installer": "pip", "packages": [ { "restraint": "EXACT", "package_version": "1.15.0", "package_name": "numpy" }, { "restraint": "EXACT", "package_version": "5.2.0", "package_name": "Pillow" } ] } ] }

apis参数代码示例 [{ "url": "/", "method": "post", "request": { "Content-type": "multipart/form-data", "data": { "type": "object", "properties": { "images": { "type": "file" } } } }, "response": { "Content-type": "applicaton/json", "data": { "type": "object", "properties": { "mnist_result": { "type": "array", "item": [ { "type": "string" } ] } } } } }]

自定义镜像类型的模型配置文件示例 模型输入和输出与目标检测模型配置文件示例类似。 模型预测输入为图片类型时，request请求示例如下： 该实例表示模型预测接收一个参数名为images、参数类型为file的预测请求，在推理界面会显示文件上传按钮，以文件形式进行预测。 1 2 3 4 5 6 7 8 9 10 11 { "Content-type": "multipart/form-data", "data": { "type": "object", "properties": { "images": { "type": "file" } } } } 模型预测输入为json数据类型时，request请求示例如下： 该实例表示模型预测接收json请求体，只有一个参数名为input、参数类型为string的预测请求，在推理界面会显示文本输入框，用于填写预测请求。 1 2 3 4 5 6 7 8 9 10 11 { "Content-type": "application/json", "data": { "type": "object", "properties": { "input": { "type": "string" } } } } 完整请求示例如下： 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 { "model_algorithm": "image_classification", "model_type": "Image", "metrics": { "f1": 0.345294, "accuracy": 0.462963, "precision": 0.338977, "recall": 0.351852 }, "apis": [{ "url": "/", "method": "post", "request": { "Content-type": "multipart/form-data", "data": { "type": "object", "properties": { "images": { "type": "file" } } } }, "response": { "Content-type": "application/json", "data": { "type": "object", "required": [ "predicted_label", "scores" ], "properties": { "predicted_label": { "type": "string" }, "scores": { "type": "array", "items": [{ "type": "array", "minItems": 2, "maxItems": 2, "items": [{ "type": "string" }, { "type": "number" } ] }] } } } } }] }

使用场景 图1 模型包应用场景 模型包应用场景： 业务人员从训练平台导入模型包：算法人员可以在训练平台上开发算法，业务人员通过联邦服务提供的Console从训练平台导入开发好的模型包。 业务人员从外部导入模型包：算法人员也可以在第三方环境上开发算法并遵循联邦学习部署服务包规范进行打包，并通过联邦服务提供的Console导入模型包。 边缘节点执行环境在本地运行模型包：边缘节点从联邦Server下载模型包之后，按照NAIE的规范解析模型包，运行模型包中的模型训练和评估算法参与联邦。 父主题： 联邦学习模型包规范