Edge Inference 분석 — Cloud vs Edge·Latency·Privacy

1
Sensor → Preprocess → Inference → Postprocess → Action
2
            5-20%      60-80%       5-15%

1
#include <arm_neon.h>
2

3
/* uint8 RGB → normalized float, 16 픽셀씩 */
4
void preprocess_neon(const uint8_t *rgb, float *out, int n_px) {
5
    float32x4_t scale = vdupq_n_f32(1.0f / 255.0f);
6
    int i = 0;
7
    for (; i + 16 <= n_px; i += 16) {
8
        uint8x16_t v8  = vld1q_u8(&rgb[i]);
9
        uint16x8_t lo  = vmovl_u8(vget_low_u8(v8));
10
        uint16x8_t hi  = vmovl_u8(vget_high_u8(v8));
11

12
        float32x4_t f0 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(lo)));
13
        float32x4_t f1 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(lo)));
14
        float32x4_t f2 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(hi)));
15
        float32x4_t f3 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(hi)));
16

17
        vst1q_f32(&out[i +  0], vmulq_f32(f0, scale));
18
        vst1q_f32(&out[i +  4], vmulq_f32(f1, scale));
19
        vst1q_f32(&out[i +  8], vmulq_f32(f2, scale));
20
        vst1q_f32(&out[i + 12], vmulq_f32(f3, scale));
21
    }
22
    for (; i < n_px; i++) out[i] = rgb[i] / 255.0f;
23
}

1
#include "tensorflow/lite/micro/micro_interpreter.h"
2

3
constexpr int kArenaSize = 64 * 1024;
4
static uint8_t tensor_arena[kArenaSize];
5

6
void run_kws(void) {
7
    const tflite::Model *model = tflite::GetModel(g_model_data);
8
    static tflite::MicroMutableOpResolver<6> resolver;
9
    resolver.AddConv2D();
10
    resolver.AddDepthwiseConv2D();
11
    resolver.AddFullyConnected();
12
    resolver.AddSoftmax();
13
    resolver.AddReshape();
14
    resolver.AddQuantize();
15

16
    static tflite::MicroInterpreter interp(
17
        model, resolver, tensor_arena, kArenaSize);
18
    interp.AllocateTensors();
19

20
    TfLiteTensor *in = interp.input(0);
21
    memcpy(in->data.int8, mfcc_buf, in->bytes);
22
    interp.Invoke();
23

24
    TfLiteTensor *out = interp.output(0);
25
    int max_idx = argmax_int8(out->data.int8, out->bytes);
26
    if (out->data.int8[max_idx] > kThreshold) trigger(max_idx);
27
}

1
#include <onnxruntime_cxx_api.h>
2

3
Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "edge");
4
Ort::SessionOptions opts;
5

6
#ifdef USE_CUDA
7
opts.AppendExecutionProvider_CUDA({});
8
#elif defined(USE_TENSORRT)
9
opts.AppendExecutionProvider_TensorRT({});
10
#elif defined(USE_NNAPI)
11
opts.AppendExecutionProvider_Nnapi(0);
12
#endif
13

14
Ort::Session session(env, "model.onnx", opts);
15

16
std::array<int64_t, 4> shape{1, 3, 224, 224};
17
auto input_tensor = Ort::Value::CreateTensor<float>(
18
    mem_info, input.data(), input.size(), shape.data(), shape.size());
19

20
auto outputs = session.Run(Ort::RunOptions{nullptr},
21
                            input_names, &input_tensor, 1,
22
                            output_names, 1);

1
typedef struct { uint8_t *raw; float *pre; box_t *det; } frame_t;
2

3
void pipeline_worker(frame_t *frames, int n) {
4
    int prep = 0, inf = 1, post = 2;
5
    while (1) {
6
        parallel_3({
7
            preprocess_neon(frames[prep].raw, frames[prep].pre, ...);
8
            inference(frames[inf].pre, frames[inf].det);
9
            postprocess_nms(frames[post].det);
10
        });
11
        int t = post; post = inf; inf = prep; prep = t;
12
    }
13
}

1
auto t0 = now(); inference(); auto t1 = now();   /* 8 ms */
2
/* 실제 카메라 → 화면은 80 ms */

1
result = cloud_api(image) if online else local_model(image)

1
서버에서 학습한 ResNet-50을 그대로 Pi에 → OOM

1
resize_scalar();   /* CPU 100% */
2
inference_gpu();   /* GPU 100%, CPU idle */

1
deploy 직전 INT8 변환 → accuracy 5% drop → 다시 train