ONNX Runtime 분석 — Execution Provider와 Cross-Platform 배포

1
import torch
2

3
model = torchvision.models.resnet50(pretrained=True)
4
model.eval()
5

6
dummy = torch.randn(1, 3, 224, 224)
7
torch.onnx.export(model, dummy, "resnet50.onnx",
8
                  input_names=['input'],
9
                  output_names=['output'],
10
                  dynamic_axes={'input': {0: 'batch'},
11
                                'output': {0: 'batch'}},
12
                  opset_version=17)

1
import onnxruntime as ort
2
import numpy as np
3

4
sess = ort.InferenceSession('resnet50.onnx',
5
                             providers=['CPUExecutionProvider'])
6

7
input = np.random.randn(1, 3, 224, 224).astype(np.float32)
8
out = sess.run(['output'], {'input': input})
9
print(out[0].shape)

1
#include <onnxruntime_cxx_api.h>
2

3
Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "test");
4
Ort::SessionOptions opts;
5
Ort::Session sess(env, "resnet50.onnx", opts);
6

7
std::vector<int64_t> shape = {1, 3, 224, 224};
8
std::vector<float> input(1*3*224*224);
9
auto mem = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
10
auto in_tensor = Ort::Value::CreateTensor<float>(mem, input.data(),
11
                                                  input.size(),
12
                                                  shape.data(), shape.size());
13

14
const char *in_name  = "input";
15
const char *out_name = "output";
16
auto outputs = sess.Run(Ort::RunOptions{}, &in_name, &in_tensor, 1, &out_name, 1);
17

18
float *out = outputs[0].GetTensorMutableData<float>();

1
# NVIDIA
2
sess = ort.InferenceSession('resnet50.onnx',
3
    providers=['TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider'])
4

5
# Apple
6
sess = ort.InferenceSession('resnet50.onnx',
7
    providers=['CoreMLExecutionProvider', 'CPUExecutionProvider'])
8

9
# Windows DirectX
10
sess = ort.InferenceSession('resnet50.onnx',
11
    providers=['DmlExecutionProvider', 'CPUExecutionProvider'])

1
options = [
2
    ('TensorrtExecutionProvider', {
3
        'trt_fp16_enable': True,
4
        'trt_int8_enable': False,
5
        'trt_engine_cache_enable': True,
6
        'trt_engine_cache_path': './trt_cache',
7
    }),
8
    'CUDAExecutionProvider',
9
    'CPUExecutionProvider',
10
]
11
sess = ort.InferenceSession('model.onnx', providers=options)

1
sess = ort.InferenceSession('model.onnx',
2
    providers=[('CoreMLExecutionProvider', {
3
        'COREML_FLAG_USE_CPU_ONLY': False,
4
        'COREML_FLAG_USE_NPU': True,
5
    })])

1
sess = ort.InferenceSession('model.onnx',
2
    providers=[('QNNExecutionProvider', {
3
        'backend_path': '/path/to/libQnnHtp.so',
4
    })])

1
ONNX Runtime 전체: ~50 MB
2
Mobile build (선택 op만): ~5 MB
3
Custom slim build: 더 작게

1
import onnxruntime as ort
2

3
sess_options = ort.SessionOptions()
4
sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
5
sess_options.optimized_model_filepath = 'model.opt.onnx'
6

7
sess = ort.InferenceSession('model.onnx', sess_options)

1
from onnxruntime.quantization import quantize_dynamic, QuantType
2

3
quantize_dynamic('model.onnx', 'model.int8.onnx',
4
                 weight_type=QuantType.QInt8)

1
from onnxruntime.quantization import quantize_static, CalibrationDataReader
2

3
class MyReader(CalibrationDataReader):
4
    def get_next(self):
5
        return {'input': next_batch()}
6

7
quantize_static('model.onnx', 'model.int8.onnx', MyReader(),
8
                quant_format=QuantFormat.QDQ)

1
import onnxruntime as ort
2

3
sess = ort.InferenceSession('whisper-tiny.onnx',
4
                             providers=['CoreMLExecutionProvider'])
5

6
mel = compute_mel_spectrogram(audio)  # (1, 80, 3000)
7
out = sess.run(None, {'input_features': mel})
8
tokens = greedy_decode(out[0])

1
import * as ort from 'onnxruntime-web';
2

3
const session = await ort.InferenceSession.create('model.onnx', {
4
    executionProviders: ['webgl', 'wasm']
5
});
6

7
const input = new ort.Tensor('float32', new Float32Array(...), [1, 3, 224, 224]);
8
const output = await session.run({input});

1
sess_options = ort.SessionOptions()
2
sess_options.enable_profiling = True
3
sess = ort.InferenceSession('model.onnx', sess_options)
4

5
# Run inference
6
sess.run(...)
7
profile_file = sess.end_profiling()
8
# Chrome Tracing format → chrome://tracing

1
providers=['CPUExecutionProvider', 'CUDAExecutionProvider']

1
Some nodes are not supported by TensorrtExecutionProvider
2
→ partition fallback to CUDA

1
TensorRT 8.4 supports opset 17
2
Model opset 18 → fail or downgrade

1
dynamic_axes={'input': {0: 'batch'}}

1
Op 'GridSample' not found