#include "kernels.hpp"

#include <algorithm>
#include <array>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <vector>

namespace {

bool VerifyGemm(int rows, int cols, int depth, bool with_bias) {
  std::vector<float> a(std::size_t(rows) * depth);
  std::vector<float> b(std::size_t(depth) * cols);
  std::vector<float> bias(static_cast<std::size_t>(cols), 0.F);
  std::vector<float> output(std::size_t(rows) * cols);
  std::vector<float> expected(std::size_t(rows) * cols);
  for (std::size_t i = 0; i < a.size(); ++i) a[i] = float(int(i % 19) - 9) * .0625F;
  for (std::size_t i = 0; i < b.size(); ++i) b[i] = float(int(i % 23) - 11) * .03125F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 7) - 3) * .125F;

  for (int row = 0; row < rows; ++row) {
    for (int col = 0; col < cols; ++col) {
      float value = with_bias ? bias[std::size_t(col)] : 0.F;
      for (int k = 0; k < depth; ++k) {
        value += a[std::size_t(row) * depth + k] * b[std::size_t(k) * cols + col];
      }
      expected[std::size_t(row) * cols + col] = value;
    }
  }
  ppocr::detail::kernels::Gemm(output.data(), a.data(), b.data(),
                                with_bias ? bias.data() : nullptr, rows, cols, depth);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 2e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "GEMM mismatch rows=" << rows << " cols=" << cols
                << " depth=" << depth << " bias=" << with_bias
                << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyIdentityRgbToNchw() {
  // Packed RGB -> NCHW BGR affine, including an odd tail so AVX-512/AVX2
  // vector bodies and the scalar remainder share one independent reference.
  constexpr int width = 37, height = 5, row_width = 40, left = 2, top = 1;
  constexpr int source_width = 64, source_height = 8;
  const float scale[3] = {1.F / (255.F * .229F), 1.F / (255.F * .224F),
                          1.F / (255.F * .225F)};
  const float shift[3] = {-.485F / .229F, -.456F / .224F, -.406F / .225F};
  std::vector<std::uint8_t> rgb(std::size_t(source_width) * source_height * 3);
  for (std::size_t i = 0; i < rgb.size(); ++i) {
    rgb[i] = static_cast<std::uint8_t>((i * 41 + 17) & 255u);
  }
  std::vector<float> output(std::size_t(3) * height * row_width, -99.F);
  std::vector<float> expected(output.size(), -99.F);
  const std::size_t plane = std::size_t(height) * row_width;
  for (int y = 0; y < height; ++y) {
    for (int x = 0; x < width; ++x) {
      const auto* pixel = rgb.data() +
          (std::size_t(top + y) * source_width + left + x) * 3;
      expected[std::size_t(y) * row_width + x] =
          static_cast<float>(pixel[2]) * scale[0] + shift[0];
      expected[plane + std::size_t(y) * row_width + x] =
          static_cast<float>(pixel[1]) * scale[1] + shift[1];
      expected[2 * plane + std::size_t(y) * row_width + x] =
          static_cast<float>(pixel[0]) * scale[2] + shift[2];
    }
  }
  ppocr::detail::kernels::WriteIdentityRgbToNchw(
      output.data(), rgb.data(), width, height, source_width, left, top,
      scale, shift, row_width);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-6F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "identity RGB->NCHW mismatch index=" << i
                << " got=" << output[i] << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyBilinearRgbToNchw(int image_width, int image_height, int left, int top,
                             int source_width, int source_height, int width,
                             int height, int row_width, bool parallel) {
  // Independent scalar bilinear + uint8 rounding + BGR affine. Covers the
  // gating detector 720x152->704x160 map, a cropped odd tail, and a 1-pixel
  // source so gather-safe SIMD tails never hide a scalar edge.
  const float scale[3] = {1.F / (255.F * .229F), 1.F / (255.F * .224F),
                          1.F / (255.F * .225F)};
  const float shift[3] = {-.485F / .229F, -.456F / .224F, -.406F / .225F};
  std::vector<std::uint8_t> rgb(std::size_t(image_width) * image_height * 3);
  for (std::size_t i = 0; i < rgb.size(); ++i) {
    rgb[i] = static_cast<std::uint8_t>((i * 41 + 17) & 255u);
  }
  std::vector<float> output(std::size_t(3) * height * row_width, -99.F);
  std::vector<float> expected(output.size(), -99.F);
  const std::size_t plane = std::size_t(height) * row_width;
  std::vector<int> x0(static_cast<std::size_t>(width));
  std::vector<int> x1(static_cast<std::size_t>(width));
  std::vector<float> dx(static_cast<std::size_t>(width));
  for (int x = 0; x < width; ++x) {
    const float fx = (float(x) + .5F) * source_width / width - .5F;
    const int x_floor = int(std::floor(fx));
    x0[static_cast<std::size_t>(x)] = std::clamp(x_floor, 0, source_width - 1);
    x1[static_cast<std::size_t>(x)] =
        std::min(x0[static_cast<std::size_t>(x)] + 1, source_width - 1);
    dx[static_cast<std::size_t>(x)] = fx - x_floor;
  }
  for (int y = 0; y < height; ++y) {
    const float fy = (float(y) + .5F) * source_height / height - .5F;
    const int y_floor = int(std::floor(fy));
    const int y0 = std::clamp(y_floor, 0, source_height - 1);
    const int y1 = std::min(y0 + 1, source_height - 1);
    const float dy = fy - y_floor;
    const float inverse_dy = 1.F - dy;
    const auto* const upper_row =
        rgb.data() + (std::size_t(top + y0) * image_width + left) * 3;
    const auto* const lower_row =
        rgb.data() + (std::size_t(top + y1) * image_width + left) * 3;
    for (int x = 0; x < width; ++x) {
      const auto* const upper_left = upper_row + std::size_t(x0[static_cast<std::size_t>(x)]) * 3;
      const auto* const upper_right = upper_row + std::size_t(x1[static_cast<std::size_t>(x)]) * 3;
      const auto* const lower_left = lower_row + std::size_t(x0[static_cast<std::size_t>(x)]) * 3;
      const auto* const lower_right = lower_row + std::size_t(x1[static_cast<std::size_t>(x)]) * 3;
      const float inverse_dx = 1.F - dx[static_cast<std::size_t>(x)];
      const float upper_red =
          float(upper_left[0]) * inverse_dx + float(upper_right[0]) * dx[static_cast<std::size_t>(x)];
      const float lower_red =
          float(lower_left[0]) * inverse_dx + float(lower_right[0]) * dx[static_cast<std::size_t>(x)];
      const float sampled_red = static_cast<float>(std::clamp(
          static_cast<int>(upper_red * inverse_dy + lower_red * dy + .5F), 0, 255));
      const float upper_green =
          float(upper_left[1]) * inverse_dx + float(upper_right[1]) * dx[static_cast<std::size_t>(x)];
      const float lower_green =
          float(lower_left[1]) * inverse_dx + float(lower_right[1]) * dx[static_cast<std::size_t>(x)];
      const float sampled_green = static_cast<float>(std::clamp(
          static_cast<int>(upper_green * inverse_dy + lower_green * dy + .5F), 0, 255));
      const float upper_blue =
          float(upper_left[2]) * inverse_dx + float(upper_right[2]) * dx[static_cast<std::size_t>(x)];
      const float lower_blue =
          float(lower_left[2]) * inverse_dx + float(lower_right[2]) * dx[static_cast<std::size_t>(x)];
      const float sampled_blue = static_cast<float>(std::clamp(
          static_cast<int>(upper_blue * inverse_dy + lower_blue * dy + .5F), 0, 255));
      expected[std::size_t(y) * row_width + x] = sampled_blue * scale[0] + shift[0];
      expected[plane + std::size_t(y) * row_width + x] = sampled_green * scale[1] + shift[1];
      expected[2 * plane + std::size_t(y) * row_width + x] = sampled_red * scale[2] + shift[2];
    }
  }
  ppocr::detail::kernels::WriteBilinearRgbToNchw(
      output.data(), rgb.data(), image_width, image_height, left, top, source_width,
      source_height, width, height, scale, shift, row_width, parallel);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-6F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "bilinear RGB->NCHW mismatch " << image_width << "x" << image_height
                << "->" << width << "x" << height << " parallel=" << parallel
                << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyLayerNorm(std::size_t rows, std::size_t width) {
  std::vector<float> input(rows * width), gamma(width), beta(width);
  std::vector<float> output(input.size()), expected(input.size());
  for (std::size_t i = 0; i < input.size(); ++i) {
    input[i] = float(int(i % 47) - 23) * .0625F;
  }
  for (std::size_t column = 0; column < width; ++column) {
    gamma[column] = float((column % 11) + 1) * .09375F;
    beta[column] = float((column % 7) - 3) * .125F;
  }
  constexpr float epsilon = 1e-5F;
  for (std::size_t row = 0; row < rows; ++row) {
    const float* src = input.data() + row * width;
    float* dst = expected.data() + row * width;
    float mean = 0.F;
    for (std::size_t column = 0; column < width; ++column) mean += src[column];
    mean /= static_cast<float>(width);
    float variance = 0.F;
    for (std::size_t column = 0; column < width; ++column) {
      const float centered = src[column] - mean;
      variance += centered * centered;
    }
    const float denom = std::sqrt(variance / static_cast<float>(width) + epsilon);
    for (std::size_t column = 0; column < width; ++column) {
      dst[column] = ((src[column] - mean) / denom) * gamma[column] + beta[column];
    }
  }
  ppocr::detail::kernels::LayerNorm(output.data(), input.data(), gamma.data(),
                                    beta.data(), rows, width, epsilon);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-6F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "LayerNorm mismatch rows=" << rows << " width=" << width
                << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifySigmoid(std::size_t n) {
  std::vector<float> input(n), output(n), expected(n);
  for (std::size_t i = 0; i < n; ++i) input[i] = float(int(i % 63) - 31) * .0625F;
  for (std::size_t i = 0; i < n; ++i) expected[i] = 1.F / (1.F + std::exp(-input[i]));
  ppocr::detail::kernels::Sigmoid(output.data(), input.data(), n);
  for (std::size_t i = 0; i < n; ++i) {
    const float tolerance = 3e-6F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "Sigmoid mismatch n=" << n << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyMaxPool2x2Same(int planes, int height, int width) {
  std::vector<float> input(std::size_t(planes) * height * width);
  std::vector<float> output(input.size());
  std::vector<float> expected(input.size());
  for (std::size_t i = 0; i < input.size(); ++i) {
    input[i] = float(int(i % 29) - 14) * .125F;
  }
  const std::size_t plane = std::size_t(height) * width;
  for (int p = 0; p < planes; ++p) {
    const float* in = input.data() + std::size_t(p) * plane;
    float* out = expected.data() + std::size_t(p) * plane;
    for (int y = 0; y + 1 < height; ++y) {
      const float* row0 = in + std::size_t(y) * width;
      const float* row1 = row0 + width;
      for (int x = 0; x + 1 < width; ++x) {
        out[std::size_t(y) * width + x] = std::max(
            std::max(row0[x], row0[x + 1]), std::max(row1[x], row1[x + 1]));
      }
      out[std::size_t(y) * width + width - 1] =
          std::max(row0[width - 1], row1[width - 1]);
    }
    if (height == 1) {
      for (int x = 0; x + 1 < width; ++x) out[x] = std::max(in[x], in[x + 1]);
    } else {
      const float* row = in + std::size_t(height - 1) * width;
      float* out_row = out + std::size_t(height - 1) * width;
      for (int x = 0; x + 1 < width; ++x) out_row[x] = std::max(row[x], row[x + 1]);
    }
    out[plane - 1] = in[plane - 1];
  }
  ppocr::detail::kernels::MaxPool2x2Same(output.data(), input.data(), planes, height, width);
  for (std::size_t i = 0; i < output.size(); ++i) {
    if (output[i] != expected[i]) {
      std::cerr << "MaxPool2x2Same mismatch at " << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyMaxPool2x2Valid(int planes, int height, int width) {
  const int out_h = height - 1, out_w = width - 1;
  std::vector<float> input(std::size_t(planes) * height * width);
  std::vector<float> output(std::size_t(planes) * out_h * out_w);
  std::vector<float> expected(output.size());
  for (std::size_t i = 0; i < input.size(); ++i) {
    input[i] = float(int(i % 29) - 14) * .125F;
  }
  for (int plane = 0; plane < planes; ++plane) {
    const float* in = input.data() + std::size_t(plane) * height * width;
    float* out = expected.data() + std::size_t(plane) * out_h * out_w;
    for (int y = 0; y < out_h; ++y) for (int x = 0; x < out_w; ++x) {
      out[std::size_t(y) * out_w + x] = std::max(
          std::max(in[std::size_t(y) * width + x], in[std::size_t(y) * width + x + 1]),
          std::max(in[std::size_t(y + 1) * width + x], in[std::size_t(y + 1) * width + x + 1]));
    }
  }
  ppocr::detail::kernels::MaxPool2x2Valid(output.data(), input.data(), planes, height, width);
  for (std::size_t i = 0; i < output.size(); ++i) {
    if (output[i] != expected[i]) {
      std::cerr << "MaxPool2x2Valid mismatch at " << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyExactGelu() {
  std::vector<float> input(4096 + 13), output(input.size()), expected(input.size());
  for (std::size_t i = 0; i < input.size(); ++i) {
    input[i] = float(int(i % 127) - 63) * .0625F;
  }
  constexpr float inv_sqrt2 = 0.7071067811865475244F;
  for (std::size_t i = 0; i < input.size(); ++i) {
    expected[i] = input[i] * .5F * (1.F + std::erf(input[i] * inv_sqrt2));
  }
  ppocr::detail::kernels::ExactGelu(output.data(), input.data(), input.size());
  for (std::size_t i = 0; i < input.size(); ++i) {
    const float tolerance = 2e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "exact GELU mismatch index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyRgbByteWiden() {
  // Covers vector blocks and an odd scalar tail. The helper is shared by the
  // x86 AVX-512/AVX2 and ARM NEON GPU-upload boundary paths.
  std::vector<std::uint8_t> input(16 * 1024 + 13);
  std::vector<float> output(input.size());
  for (std::size_t index = 0; index < input.size(); ++index) {
    input[index] = static_cast<std::uint8_t>((index * 37 + 19) & 255u);
  }
  ppocr::detail::kernels::WidenU8ToFloat(output.data(), input.data(), input.size());
  for (std::size_t index = 0; index < input.size(); ++index) {
    if (output[index] != static_cast<float>(input[index])) {
      std::cerr << "RGB byte widen mismatch index=" << index << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConv3x3Tile(int input_channels, int output_channels, int height,
                       int width, bool relu) {
  const std::size_t input_plane = std::size_t(height) * width;
  const std::size_t output_plane = input_plane;
  std::vector<float> input(std::size_t(input_channels) * input_plane);
  std::vector<float> weights(std::size_t(output_channels) * input_channels * 9);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> output(std::size_t(output_channels) * output_plane);
  std::vector<float> expected(output.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 31) - 15) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 13) - 6) * .0625F;
  for (int output_channel = 0; output_channel < output_channels; ++output_channel) {
    for (int y = 0; y < height; ++y) {
      for (int x = 0; x < width; ++x) {
        float value = bias[std::size_t(output_channel)];
        for (int input_channel = 0; input_channel < input_channels; ++input_channel) {
          const float* plane = input.data() + std::size_t(input_channel) * input_plane;
          const float* filter = weights.data() +
              (std::size_t(output_channel) * input_channels + input_channel) * 9;
          for (int ky = 0; ky < 3; ++ky) for (int kx = 0; kx < 3; ++kx) {
            const int iy = y + ky - 1, ix = x + kx - 1;
            if (iy >= 0 && iy < height && ix >= 0 && ix < width) {
              value += plane[std::size_t(iy) * width + ix] * filter[ky * 3 + kx];
            }
          }
        }
        expected[(std::size_t(output_channel) * height + y) * width + x] =
            relu ? std::max(value, 0.F) : value;
      }
    }
  }
  ppocr::detail::kernels::Conv2d(output.data(), input.data(), weights.data(), bias.data(),
                                  input_channels, output_channels, height, width,
                                  height, width, 3, 3, 1, 1, 1, 1, relu);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "3x3 convolution mismatch C=" << input_channels << " M="
                << output_channels << " H=" << height << " W=" << width
                << " relu=" << relu << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConv3x3Stride2(int input_channels, int output_channels, int height,
                          int width, bool relu) {
  const int pad = 1;
  const int output_height = (height + 2 * pad - 3) / 2 + 1;
  const int output_width = (width + 2 * pad - 3) / 2 + 1;
  const std::size_t input_plane = std::size_t(height) * width;
  const std::size_t output_plane = std::size_t(output_height) * output_width;
  std::vector<float> input(std::size_t(input_channels) * input_plane);
  std::vector<float> weights(std::size_t(output_channels) * input_channels * 9);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> output(std::size_t(output_channels) * output_plane);
  std::vector<float> expected(output.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 31) - 15) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 13) - 6) * .0625F;
  for (int output_channel = 0; output_channel < output_channels; ++output_channel) {
    for (int y = 0; y < output_height; ++y) {
      for (int x = 0; x < output_width; ++x) {
        float value = bias[std::size_t(output_channel)];
        for (int input_channel = 0; input_channel < input_channels; ++input_channel) {
          const float* plane = input.data() + std::size_t(input_channel) * input_plane;
          const float* filter = weights.data() +
              (std::size_t(output_channel) * input_channels + input_channel) * 9;
          for (int ky = 0; ky < 3; ++ky) for (int kx = 0; kx < 3; ++kx) {
            const int iy = y * 2 + ky - pad, ix = x * 2 + kx - pad;
            if (iy >= 0 && iy < height && ix >= 0 && ix < width) {
              value += plane[std::size_t(iy) * width + ix] * filter[ky * 3 + kx];
            }
          }
        }
        expected[(std::size_t(output_channel) * output_height + y) * output_width + x] =
            relu ? std::max(value, 0.F) : value;
      }
    }
  }
  ppocr::detail::kernels::Conv2d(output.data(), input.data(), weights.data(), bias.data(),
                                  input_channels, output_channels, height, width,
                                  output_height, output_width, 3, 3, 2, 2, pad, pad, relu);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "3x3 stride-2 mismatch C=" << input_channels << " M="
                << output_channels << " H=" << height << " W=" << width
                << " relu=" << relu << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConcatChannelConv(int source_count, int channels_each, int output_channels,
                             int height, int width, bool relu,
                             int stride_h = 1, int stride_w = 1) {
  const int total_channels = source_count * channels_each;
  const int pad = 1;
  const int output_height = (height + 2 * pad - 3) / stride_h + 1;
  const int output_width = (width + 2 * pad - 3) / stride_w + 1;
  const std::size_t plane = std::size_t(height) * width;
  const std::size_t output_plane = std::size_t(output_height) * output_width;
  std::vector<std::vector<float>> sources(static_cast<std::size_t>(source_count));
  std::vector<const float*> source_ptrs(static_cast<std::size_t>(source_count));
  std::vector<int> source_channels(static_cast<std::size_t>(source_count), channels_each);
  std::vector<float> concat(std::size_t(total_channels) * plane);
  for (int source = 0; source < source_count; ++source) {
    sources[static_cast<std::size_t>(source)].resize(std::size_t(channels_each) * plane);
    for (std::size_t i = 0; i < sources[static_cast<std::size_t>(source)].size(); ++i) {
      sources[static_cast<std::size_t>(source)][i] =
          float(int((i + std::size_t(source) * 17) % 31) - 15) * .03125F;
    }
    source_ptrs[static_cast<std::size_t>(source)] = sources[static_cast<std::size_t>(source)].data();
    std::memcpy(concat.data() + std::size_t(source) * channels_each * plane,
                sources[static_cast<std::size_t>(source)].data(),
                sources[static_cast<std::size_t>(source)].size() * sizeof(float));
  }
  std::vector<float> weights(std::size_t(output_channels) * total_channels * 9);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> expected(std::size_t(output_channels) * output_plane);
  std::vector<float> output(expected.size());
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 13) - 6) * .0625F;
  for (int output_channel = 0; output_channel < output_channels; ++output_channel) {
    for (int y = 0; y < output_height; ++y) for (int x = 0; x < output_width; ++x) {
      float value = bias[static_cast<std::size_t>(output_channel)];
      for (int input_channel = 0; input_channel < total_channels; ++input_channel) {
        const float* src = concat.data() + std::size_t(input_channel) * plane;
        const float* filter = weights.data() +
            (std::size_t(output_channel) * total_channels + input_channel) * 9;
        for (int ky = 0; ky < 3; ++ky) for (int kx = 0; kx < 3; ++kx) {
          const int iy = y * stride_h + ky - pad, ix = x * stride_w + kx - pad;
          if (iy >= 0 && iy < height && ix >= 0 && ix < width) {
            value += src[std::size_t(iy) * width + ix] * filter[ky * 3 + kx];
          }
        }
      }
      expected[(std::size_t(output_channel) * output_height + y) * output_width + x] =
          relu ? std::max(value, 0.F) : value;
    }
  }
  ppocr::detail::kernels::ConcatChannelConv2d(
      output.data(), source_ptrs.data(), source_channels.data(), source_count,
      weights.data(), bias.data(), output_channels, height, width, output_height,
      output_width, 3, 3, stride_h, stride_w, pad, pad, relu);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "concat-channel conv mismatch sources=" << source_count
                << " C=" << channels_each << " M=" << output_channels
                << " stride=" << stride_h << "x" << stride_w
                << " relu=" << relu << " index=" << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConcatChannelConvMixed(const int* channels, int source_count,
                                  int output_channels, int height, int width,
                                  bool relu, int stride_h, int stride_w) {
  const int pad = 1;
  const int output_height = (height + 2 * pad - 3) / stride_h + 1;
  const int output_width = (width + 2 * pad - 3) / stride_w + 1;
  const std::size_t plane = std::size_t(height) * width;
  const std::size_t output_plane = std::size_t(output_height) * output_width;
  int total_channels = 0;
  for (int source = 0; source < source_count; ++source) total_channels += channels[source];
  std::vector<std::vector<float>> sources(static_cast<std::size_t>(source_count));
  std::vector<const float*> source_ptrs(static_cast<std::size_t>(source_count));
  std::vector<int> source_channels(static_cast<std::size_t>(source_count));
  std::vector<float> concat(std::size_t(total_channels) * plane);
  int channel0 = 0;
  for (int source = 0; source < source_count; ++source) {
    source_channels[static_cast<std::size_t>(source)] = channels[source];
    sources[static_cast<std::size_t>(source)].resize(std::size_t(channels[source]) * plane);
    for (std::size_t i = 0; i < sources[static_cast<std::size_t>(source)].size(); ++i) {
      sources[static_cast<std::size_t>(source)][i] =
          float(int((i + std::size_t(source) * 17) % 31) - 15) * .03125F;
    }
    source_ptrs[static_cast<std::size_t>(source)] = sources[static_cast<std::size_t>(source)].data();
    std::memcpy(concat.data() + std::size_t(channel0) * plane,
                sources[static_cast<std::size_t>(source)].data(),
                sources[static_cast<std::size_t>(source)].size() * sizeof(float));
    channel0 += channels[source];
  }
  std::vector<float> weights(std::size_t(output_channels) * total_channels * 9);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> expected(std::size_t(output_channels) * output_plane);
  std::vector<float> output(expected.size());
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 13) - 6) * .0625F;
  for (int output_channel = 0; output_channel < output_channels; ++output_channel) {
    for (int y = 0; y < output_height; ++y) for (int x = 0; x < output_width; ++x) {
      float value = bias[static_cast<std::size_t>(output_channel)];
      for (int input_channel = 0; input_channel < total_channels; ++input_channel) {
        const float* src = concat.data() + std::size_t(input_channel) * plane;
        const float* filter = weights.data() +
            (std::size_t(output_channel) * total_channels + input_channel) * 9;
        for (int ky = 0; ky < 3; ++ky) for (int kx = 0; kx < 3; ++kx) {
          const int iy = y * stride_h + ky - pad, ix = x * stride_w + kx - pad;
          if (iy >= 0 && iy < height && ix >= 0 && ix < width) {
            value += src[std::size_t(iy) * width + ix] * filter[ky * 3 + kx];
          }
        }
      }
      expected[(std::size_t(output_channel) * output_height + y) * output_width + x] =
          relu ? std::max(value, 0.F) : value;
    }
  }
  ppocr::detail::kernels::ConcatChannelConv2d(
      output.data(), source_ptrs.data(), source_channels.data(), source_count,
      weights.data(), bias.data(), output_channels, height, width, output_height,
      output_width, 3, 3, stride_h, stride_w, pad, pad, relu);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "mixed concat-channel conv mismatch index=" << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConcatConvDualTranspose(int source_count, int channels_each, int conv_oc,
                                   int final_oc, int height, int width) {
  const int pad = 1;
  const int conv_h = height;
  const int conv_width = width;
  const int mid_h = conv_h * 2, mid_w = conv_width * 2;
  const int out_h = conv_h * 4, out_w = conv_width * 4;
  const std::size_t plane = std::size_t(height) * width;
  const int total_channels = source_count * channels_each;
  std::vector<std::vector<float>> sources(static_cast<std::size_t>(source_count));
  std::vector<const float*> source_ptrs(static_cast<std::size_t>(source_count));
  std::vector<int> source_channels(static_cast<std::size_t>(source_count), channels_each);
  for (int source = 0; source < source_count; ++source) {
    sources[static_cast<std::size_t>(source)].resize(std::size_t(channels_each) * plane);
    for (std::size_t i = 0; i < sources[static_cast<std::size_t>(source)].size(); ++i) {
      sources[static_cast<std::size_t>(source)][i] =
          float(int((i + std::size_t(source) * 13) % 29) - 14) * .03125F;
    }
    source_ptrs[static_cast<std::size_t>(source)] = sources[static_cast<std::size_t>(source)].data();
  }
  std::vector<float> conv_weights(std::size_t(conv_oc) * total_channels * 9);
  std::vector<float> conv_b(static_cast<std::size_t>(conv_oc));
  std::vector<float> ct0_w(std::size_t(conv_oc) * conv_oc * 4);
  std::vector<float> ct0_b(static_cast<std::size_t>(conv_oc));
  std::vector<float> ct1_w(std::size_t(conv_oc) * final_oc * 4);
  std::vector<float> ct1_b(static_cast<std::size_t>(final_oc));
  for (std::size_t i = 0; i < conv_weights.size(); ++i)
    conv_weights[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < conv_b.size(); ++i) conv_b[i] = float(int(i % 13) - 6) * .0625F;
  for (std::size_t i = 0; i < ct0_w.size(); ++i) ct0_w[i] = float(int(i % 23) - 11) * .03125F;
  for (std::size_t i = 0; i < ct0_b.size(); ++i) ct0_b[i] = float(int(i % 7) - 3) * .0625F;
  for (std::size_t i = 0; i < ct1_w.size(); ++i) ct1_w[i] = float(int(i % 19) - 9) * .03125F;
  for (std::size_t i = 0; i < ct1_b.size(); ++i) ct1_b[i] = float(int(i % 5) - 2) * .0625F;
  std::vector<float> concat(std::size_t(total_channels) * plane);
  for (int source = 0; source < source_count; ++source) {
    std::memcpy(concat.data() + std::size_t(source * channels_each) * plane,
                sources[static_cast<std::size_t>(source)].data(),
                sources[static_cast<std::size_t>(source)].size() * sizeof(float));
  }
  std::vector<float> conv_ref(std::size_t(conv_oc) * conv_h * conv_width);
  for (int oc = 0; oc < conv_oc; ++oc) {
    for (int y = 0; y < conv_h; ++y) for (int x = 0; x < conv_width; ++x) {
      float value = conv_b[static_cast<std::size_t>(oc)];
      for (int ic = 0; ic < total_channels; ++ic) {
        const float* src = concat.data() + std::size_t(ic) * plane;
        const float* filter =
            conv_weights.data() + (std::size_t(oc) * total_channels + ic) * 9;
        for (int ky = 0; ky < 3; ++ky) for (int kx = 0; kx < 3; ++kx) {
          const int iy = y + ky - pad, ix = x + kx - pad;
          if (iy >= 0 && iy < height && ix >= 0 && ix < width)
            value += src[std::size_t(iy) * width + ix] * filter[ky * 3 + kx];
        }
      }
      conv_ref[(std::size_t(oc) * conv_h + y) * conv_width + x] = std::max(value, 0.F);
    }
  }
  std::vector<float> mid_ref(std::size_t(conv_oc) * mid_h * mid_w, 0.F);
  const std::size_t conv_plane = std::size_t(conv_h) * conv_width;
  const std::size_t mid_plane = std::size_t(mid_h) * mid_w;
  for (int oc = 0; oc < conv_oc; ++oc) {
    float* out = mid_ref.data() + std::size_t(oc) * mid_plane;
    std::fill_n(out, mid_plane, ct0_b[static_cast<std::size_t>(oc)]);
    for (int ic = 0; ic < conv_oc; ++ic) {
      const float* src = conv_ref.data() + std::size_t(ic) * conv_plane;
      const float* w = ct0_w.data() + (std::size_t(ic) * conv_oc + oc) * 4;
      for (int y = 0; y < conv_h; ++y) {
        const float* row = src + std::size_t(y) * conv_width;
        float* dst0 = out + std::size_t(2 * y) * mid_w;
        float* dst1 = dst0 + mid_w;
        for (int x = 0; x < conv_width; ++x) {
          const float value = row[x];
          dst0[2 * x] += value * w[0]; dst0[2 * x + 1] += value * w[1];
          dst1[2 * x] += value * w[2]; dst1[2 * x + 1] += value * w[3];
        }
      }
    }
    for (std::size_t i = 0; i < mid_plane; ++i) out[i] = std::max(out[i], 0.F);
  }
  std::vector<float> expected(std::size_t(final_oc) * out_h * out_w, 0.F);
  const std::size_t out_plane = std::size_t(out_h) * out_w;
  for (int oc = 0; oc < final_oc; ++oc) {
    float* out = expected.data() + std::size_t(oc) * out_plane;
    std::fill_n(out, out_plane, ct1_b[static_cast<std::size_t>(oc)]);
    for (int ic = 0; ic < conv_oc; ++ic) {
      const float* src = mid_ref.data() + std::size_t(ic) * mid_plane;
      const float* w = ct1_w.data() + (std::size_t(ic) * final_oc + oc) * 4;
      for (int y = 0; y < mid_h; ++y) {
        const float* row = src + std::size_t(y) * mid_w;
        float* dst0 = out + std::size_t(2 * y) * out_w;
        float* dst1 = dst0 + out_w;
        for (int x = 0; x < mid_w; ++x) {
          const float value = row[x];
          dst0[2 * x] += value * w[0]; dst0[2 * x + 1] += value * w[1];
          dst1[2 * x] += value * w[2]; dst1[2 * x + 1] += value * w[3];
        }
      }
    }
    for (std::size_t i = 0; i < out_plane; ++i) out[i] = 1.F / (1.F + std::exp(-out[i]));
  }
  std::vector<float> conv(std::size_t(conv_oc) * conv_h * conv_width);
  std::vector<float> mid(std::size_t(conv_oc) * mid_h * mid_w);
  std::vector<float> output(expected.size());
  ppocr::detail::kernels::ConcatChannelConv2d(
      conv.data(), source_ptrs.data(), source_channels.data(), source_count, conv_weights.data(),
      conv_b.data(), conv_oc, height, width, conv_h, conv_width, 3, 3, 1, 1, pad, pad, true);
  ppocr::detail::kernels::ConvTranspose2x2(mid.data(), conv.data(), ct0_w.data(), ct0_b.data(),
                                           conv_oc, conv_oc, conv_h, conv_width, 1);
  ppocr::detail::kernels::ConvTranspose2x2(output.data(), mid.data(), ct1_w.data(),
                                           ct1_b.data(), conv_oc, final_oc, mid_h, mid_w, 2);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 4e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "concat-conv dual transpose mismatch H=" << height << " W=" << width
                << " index=" << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyDetStemFromNchw(int height, int width) {
  const int c0_h = (height + 2 - 3) / 2 + 1;
  const int c0_w = (width + 2 - 3) / 2 + 1;
  const int stem_h = (c0_h + 2 - 3) / 2 + 1;
  const int stem_w = (c0_w + 2 - 3) / 2 + 1;
  const int oc0 = 16, oc1 = 8, oc2 = 16, ocs = 16;
  std::vector<float> rgb(std::size_t(3) * height * width);
  std::vector<float> w0(std::size_t(oc0) * 3 * 9), b0(oc0);
  std::vector<float> w1(std::size_t(oc1) * oc0 * 4), b1(oc1);
  std::vector<float> w2(std::size_t(oc2) * oc1 * 4), b2(oc2);
  std::vector<float> ws(std::size_t(ocs) * (oc0 + oc2) * 9), bs(ocs);
  for (std::size_t i = 0; i < rgb.size(); ++i) rgb[i] = float(int(i % 31) - 15) * .03125F;
  for (std::size_t i = 0; i < w0.size(); ++i) w0[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < b0.size(); ++i) b0[i] = float(int(i % 13) - 6) * .0625F;
  for (std::size_t i = 0; i < w1.size(); ++i) w1[i] = float(int(i % 23) - 11) * .015625F;
  for (std::size_t i = 0; i < b1.size(); ++i) b1[i] = float(int(i % 11) - 5) * .0625F;
  for (std::size_t i = 0; i < w2.size(); ++i) w2[i] = float(int(i % 19) - 9) * .015625F;
  for (std::size_t i = 0; i < b2.size(); ++i) b2[i] = float(int(i % 7) - 3) * .0625F;
  for (std::size_t i = 0; i < ws.size(); ++i) ws[i] = float(int(i % 17) - 8) * .015625F;
  for (std::size_t i = 0; i < bs.size(); ++i) bs[i] = float(int(i % 5) - 2) * .0625F;
  std::vector<float> c0(std::size_t(oc0) * c0_h * c0_w);
  std::vector<float> c1(std::size_t(oc1) * c0_h * c0_w);
  std::vector<float> c2(std::size_t(oc2) * c0_h * c0_w);
  std::vector<float> pooled(c0.size());
  std::vector<float> expected(std::size_t(ocs) * stem_h * stem_w);
  std::vector<float> output(expected.size(), -99.F);
  ppocr::detail::kernels::Conv2d(c0.data(), rgb.data(), w0.data(), b0.data(), 3, oc0, height,
                                 width, c0_h, c0_w, 3, 3, 2, 2, 1, 1, true);
  ppocr::detail::kernels::Conv2d(c1.data(), c0.data(), w1.data(), b1.data(), oc0, oc1, c0_h,
                                 c0_w, c0_h, c0_w, 2, 2, 1, 1, 0, 0, true);
  ppocr::detail::kernels::Conv2d(c2.data(), c1.data(), w2.data(), b2.data(), oc1, oc2, c0_h,
                                 c0_w, c0_h, c0_w, 2, 2, 1, 1, 0, 0, true);
  ppocr::detail::kernels::MaxPool2x2Same(pooled.data(), c0.data(), std::size_t(oc0), c0_h, c0_w);
  const float* sources[2] = {pooled.data(), c2.data()};
  const int chans[2] = {oc0, oc2};
  ppocr::detail::kernels::ConcatChannelConv2d(expected.data(), sources, chans, 2, ws.data(),
                                              bs.data(), ocs, c0_h, c0_w, stem_h, stem_w, 3, 3,
                                              2, 2, 1, 1, true);
  ppocr::detail::kernels::DetStemFromNchw(output.data(), rgb.data(), w0.data(), b0.data(), oc0,
                                          w1.data(), b1.data(), oc1, w2.data(), b2.data(), oc2,
                                          ws.data(), bs.data(), ocs, height, width, true, true);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "det stem mismatch H=" << height << " W=" << width << " index=" << i
                << " got=" << output[i] << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConv2x2SameUpper(int input_channels, int output_channels, int height,
                            int width, bool relu) {
  const std::size_t plane = std::size_t(height) * width;
  std::vector<float> input(std::size_t(input_channels) * plane);
  std::vector<float> weights(std::size_t(output_channels) * input_channels * 4);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> output(std::size_t(output_channels) * plane);
  std::vector<float> expected(output.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 13) - 6) * .0625F;
  for (int output_channel = 0; output_channel < output_channels; ++output_channel) {
    for (int y = 0; y < height; ++y) for (int x = 0; x < width; ++x) {
      float value = bias[std::size_t(output_channel)];
      for (int input_channel = 0; input_channel < input_channels; ++input_channel) {
        const float* source = input.data() + std::size_t(input_channel) * plane;
        const float* kernel = weights.data() +
            (std::size_t(output_channel) * input_channels + input_channel) * 4;
        for (int ky = 0; ky < 2; ++ky) for (int kx = 0; kx < 2; ++kx) {
          const int iy = y + ky, ix = x + kx;
          if (iy < height && ix < width) {
            value += source[std::size_t(iy) * width + ix] * kernel[ky * 2 + kx];
          }
        }
      }
      expected[(std::size_t(output_channel) * height + y) * width + x] =
          relu ? std::max(value, 0.F) : value;
    }
  }
  ppocr::detail::kernels::Conv2d(output.data(), input.data(), weights.data(), bias.data(),
                                  input_channels, output_channels, height, width,
                                  height, width, 2, 2, 1, 1, 0, 0, relu);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "2x2 SAME_UPPER mismatch C=" << input_channels << " M="
                << output_channels << " H=" << height << " W=" << width
                << " relu=" << relu << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConvWideTile(int input_channels, int output_channels, int height,
                        int width, int kernel) {
  const int pad = kernel / 2;
  const std::size_t input_plane = std::size_t(height) * width;
  std::vector<float> input(std::size_t(input_channels) * input_plane);
  std::vector<float> weights(std::size_t(output_channels) * input_channels * kernel * kernel);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> output(std::size_t(output_channels) * input_plane);
  std::vector<float> expected(output.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 31) - 15) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 13) - 6) * .0625F;
  for (int output_channel = 0; output_channel < output_channels; ++output_channel) {
    for (int y = 0; y < height; ++y) for (int x = 0; x < width; ++x) {
      float value = bias[std::size_t(output_channel)];
      for (int input_channel = 0; input_channel < input_channels; ++input_channel) {
        const float* plane = input.data() + std::size_t(input_channel) * input_plane;
        const float* filter = weights.data() +
            (std::size_t(output_channel) * input_channels + input_channel) * kernel * kernel;
        for (int ky = 0; ky < kernel; ++ky) for (int kx = 0; kx < kernel; ++kx) {
          const int iy = y + ky - pad, ix = x + kx - pad;
          if (iy >= 0 && iy < height && ix >= 0 && ix < width) {
            value += plane[std::size_t(iy) * width + ix] * filter[ky * kernel + kx];
          }
        }
      }
      expected[(std::size_t(output_channel) * height + y) * width + x] = value;
    }
  }
  ppocr::detail::kernels::Conv2d(output.data(), input.data(), weights.data(), bias.data(),
                                  input_channels, output_channels, height, width,
                                  height, width, kernel, kernel, 1, 1, pad, pad, false);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "wide convolution mismatch K=" << kernel << " C=" << input_channels
                << " M=" << output_channels << " H=" << height << " W=" << width
                << " index=" << i << " got=" << output[i] << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyPointwiseConv(int input_channels, int output_channels, std::size_t plane) {
  std::vector<float> input(std::size_t(input_channels) * plane);
  std::vector<float> weights(std::size_t(output_channels) * input_channels);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> expected(std::size_t(output_channels) * plane);
  std::vector<float> output(expected.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 37) - 18) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 31) - 15) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 17) - 8) * .0625F;
  for (int output_channel = 0; output_channel < output_channels; ++output_channel) {
    float* dst = expected.data() + std::size_t(output_channel) * plane;
    const float* filter = weights.data() + std::size_t(output_channel) * input_channels;
    const float base = bias[static_cast<std::size_t>(output_channel)];
    for (std::size_t index = 0; index < plane; ++index) {
      float value = base;
      for (int input_channel = 0; input_channel < input_channels; ++input_channel) {
        value += filter[input_channel] *
            input[std::size_t(input_channel) * plane + index];
      }
      dst[index] = value;
    }
  }
  ppocr::detail::kernels::PointwiseConv(output.data(), input.data(), weights.data(),
                                         bias.data(), output_channels, input_channels, plane);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "pointwise conv mismatch C=" << input_channels << " M="
                << output_channels << " plane=" << plane << " index=" << i
                << " got=" << output[i] << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyPointwiseConvRelu(int input_channels, int output_channels, std::size_t plane) {
  std::vector<float> input(std::size_t(input_channels) * plane);
  std::vector<float> weights(std::size_t(output_channels) * input_channels);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> expected(std::size_t(output_channels) * plane);
  std::vector<float> output(expected.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 37) - 18) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 31) - 15) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 17) - 8) * .0625F;
  ppocr::detail::kernels::PointwiseConv(expected.data(), input.data(), weights.data(),
                                         bias.data(), output_channels, input_channels, plane);
  ppocr::detail::kernels::Relu(expected.data(), expected.data(), expected.size());
  ppocr::detail::kernels::PointwiseConvRelu(output.data(), input.data(), weights.data(),
                                            bias.data(), output_channels, input_channels, plane);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "pointwise conv relu mismatch C=" << input_channels << " M="
                << output_channels << " plane=" << plane << " index=" << i
                << " got=" << output[i] << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyExpandGeluProjectAdd(int channels, int hidden, std::size_t plane) {
  std::vector<float> input(std::size_t(channels) * plane);
  std::vector<float> expand_weights(std::size_t(hidden) * channels);
  std::vector<float> expand_bias(static_cast<std::size_t>(hidden));
  std::vector<float> project_weights(std::size_t(channels) * hidden);
  std::vector<float> project_bias(static_cast<std::size_t>(channels));
  std::vector<float> expected(input.size());
  std::vector<float> output(input.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 37) - 18) * .03125F;
  for (std::size_t i = 0; i < expand_weights.size(); ++i)
    expand_weights[i] = float(int(i % 31) - 15) * .015625F;
  for (std::size_t i = 0; i < expand_bias.size(); ++i)
    expand_bias[i] = float(int(i % 17) - 8) * .0625F;
  for (std::size_t i = 0; i < project_weights.size(); ++i)
    project_weights[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < project_bias.size(); ++i)
    project_bias[i] = float(int(i % 13) - 6) * .0625F;
  constexpr float inv_sqrt2 = 0.7071067811865475244F;
  std::vector<float> hidden_act(static_cast<std::size_t>(hidden));
  for (std::size_t spatial = 0; spatial < plane; ++spatial) {
    for (int hidden_channel = 0; hidden_channel < hidden; ++hidden_channel) {
      float sum = expand_bias[static_cast<std::size_t>(hidden_channel)];
      const float* filter = expand_weights.data() + std::size_t(hidden_channel) * channels;
      for (int channel = 0; channel < channels; ++channel) {
        sum += filter[channel] * input[std::size_t(channel) * plane + spatial];
      }
      hidden_act[static_cast<std::size_t>(hidden_channel)] =
          sum * .5F * (1.F + std::erf(sum * inv_sqrt2));
    }
    for (int channel = 0; channel < channels; ++channel) {
      float sum = project_bias[static_cast<std::size_t>(channel)] +
                  input[std::size_t(channel) * plane + spatial];
      const float* filter = project_weights.data() + std::size_t(channel) * hidden;
      for (int hidden_channel = 0; hidden_channel < hidden; ++hidden_channel) {
        sum += filter[hidden_channel] * hidden_act[static_cast<std::size_t>(hidden_channel)];
      }
      expected[std::size_t(channel) * plane + spatial] = sum;
    }
  }
  ppocr::detail::kernels::ExpandGeluProjectAdd(
      output.data(), input.data(), expand_weights.data(), expand_bias.data(),
      project_weights.data(), project_bias.data(), channels, hidden, plane);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 2e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "expand-GELU-project-add mismatch C=" << channels << " H="
                << hidden << " plane=" << plane << " index=" << i
                << " got=" << output[i] << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyPointwiseResidualBatch(int batches, int input_channels,
                                  int output_channels, std::size_t plane,
                                  int activation) {
  const std::size_t input_batch = std::size_t(input_channels) * plane;
  const std::size_t output_batch = std::size_t(output_channels) * plane;
  std::vector<float> input(std::size_t(batches) * input_batch);
  std::vector<float> weights(std::size_t(output_channels) * input_channels);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> residual(std::size_t(batches) * output_batch);
  std::vector<float> expected(residual.size());
  std::vector<float> output(residual.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 37) - 18) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 31) - 15) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 17) - 8) * .0625F;
  for (std::size_t i = 0; i < residual.size(); ++i) residual[i] = float(int(i % 29) - 14) * .03125F;

  for (int batch = 0; batch < batches; ++batch) {
    float* dst = expected.data() + std::size_t(batch) * output_batch;
    const float* src = input.data() + std::size_t(batch) * input_batch;
    const float* add = residual.data() + std::size_t(batch) * output_batch;
    if (activation == 0) {
      ppocr::detail::kernels::PointwiseConvAdd(dst, src, weights.data(), bias.data(), add,
                                                output_channels, input_channels, plane);
    } else if (activation == 1) {
      ppocr::detail::kernels::PointwiseConvAddRelu(dst, src, weights.data(), bias.data(), add,
                                                    output_channels, input_channels, plane);
    } else {
      ppocr::detail::kernels::PointwiseConvAddSwish(dst, src, weights.data(), bias.data(), add,
                                                     output_channels, input_channels, plane);
    }
  }
  if (activation == 0) {
    ppocr::detail::kernels::PointwiseConvAddBatch(output.data(), input.data(), weights.data(),
                                                   bias.data(), residual.data(), batches,
                                                   output_channels, input_channels, plane);
  } else if (activation == 1) {
    ppocr::detail::kernels::PointwiseConvAddReluBatch(output.data(), input.data(), weights.data(),
                                                       bias.data(), residual.data(), batches,
                                                       output_channels, input_channels, plane);
  } else {
    ppocr::detail::kernels::PointwiseConvAddSwishBatch(output.data(), input.data(), weights.data(),
                                                        bias.data(), residual.data(), batches,
                                                        output_channels, input_channels, plane);
  }
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "batched residual pointwise mismatch N=" << batches << " C="
                << input_channels << " M=" << output_channels << " plane=" << plane
                << " activation=" << activation << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyDepthwisePointwiseFused(int channels, int output_channels, int height,
                                   int width, int activation) {
  const std::size_t in_n = std::size_t(channels) * height * width;
  const std::size_t out_n = std::size_t(output_channels) * height * width;
  std::vector<float> input(in_n), dw_w(std::size_t(channels) * 9);
  std::vector<float> dw_b(static_cast<std::size_t>(channels));
  std::vector<float> pw_w(std::size_t(output_channels) * channels);
  std::vector<float> pw_b(static_cast<std::size_t>(output_channels));
  std::vector<float> intermediate(in_n), expected(out_n), output(out_n, -99.F);
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < dw_w.size(); ++i) dw_w[i] = float(int(i % 23) - 11) * .03125F;
  for (std::size_t i = 0; i < dw_b.size(); ++i) dw_b[i] = float(int(i % 13) - 6) * .0625F;
  for (std::size_t i = 0; i < pw_w.size(); ++i) pw_w[i] = float(int(i % 31) - 15) * .015625F;
  for (std::size_t i = 0; i < pw_b.size(); ++i) pw_b[i] = float(int(i % 17) - 8) * .0625F;
  ppocr::detail::kernels::DepthwiseConv(
      intermediate.data(), input.data(), dw_w.data(), dw_b.data(), channels, height,
      width, height, width, 3, 3, 1, 1, 1, 1);
  if (activation == 1) {
    ppocr::detail::kernels::PointwiseConvRelu(expected.data(), intermediate.data(),
                                              pw_w.data(), pw_b.data(), output_channels,
                                              channels, std::size_t(height) * width);
  } else if (activation == 2) {
    ppocr::detail::kernels::PointwiseConvHardSwish(expected.data(), intermediate.data(),
                                                   pw_w.data(), pw_b.data(), output_channels,
                                                   channels, std::size_t(height) * width);
  } else {
    ppocr::detail::kernels::PointwiseConv(expected.data(), intermediate.data(), pw_w.data(),
                                          pw_b.data(), output_channels, channels,
                                          std::size_t(height) * width);
    if (activation == 3) {
      ppocr::detail::kernels::Gelu(expected.data(), expected.data(), expected.size(),
                                   expected.size());
    }
  }
  const bool fused = ppocr::detail::kernels::DepthwisePointwiseConvFused(
      output.data(), input.data(), dw_w.data(), dw_b.data(), pw_w.data(), pw_b.data(),
      1, channels, output_channels, height, width, 3, 3, 1, 1, 1, 1, activation);
  if (!fused) {
    // Scalar/AVX2 hosts skip the fused AVX-512 path; the two-pass kernels
    // remain the production fallback and are already covered elsewhere.
    return true;
  }
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 5e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "fused DW+PW mismatch C=" << channels << " M=" << output_channels
                << " H=" << height << " W=" << width << " act=" << activation
                << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyDepthwisePointwiseFused5x5(int channels, int output_channels, int height,
                                      int width, int activation) {
  const std::size_t in_n = std::size_t(channels) * height * width;
  const std::size_t out_n = std::size_t(output_channels) * height * width;
  std::vector<float> input(in_n), dw_w(std::size_t(channels) * 25);
  std::vector<float> dw_b(static_cast<std::size_t>(channels));
  std::vector<float> pw_w(std::size_t(output_channels) * channels);
  std::vector<float> pw_b(static_cast<std::size_t>(output_channels));
  std::vector<float> intermediate(in_n), expected(out_n), output(out_n, -99.F);
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < dw_w.size(); ++i) dw_w[i] = float(int(i % 23) - 11) * .03125F;
  for (std::size_t i = 0; i < dw_b.size(); ++i) dw_b[i] = float(int(i % 13) - 6) * .0625F;
  for (std::size_t i = 0; i < pw_w.size(); ++i) pw_w[i] = float(int(i % 31) - 15) * .015625F;
  for (std::size_t i = 0; i < pw_b.size(); ++i) pw_b[i] = float(int(i % 17) - 8) * .0625F;
  ppocr::detail::kernels::DepthwiseConv(
      intermediate.data(), input.data(), dw_w.data(), dw_b.data(), channels, height,
      width, height, width, 5, 5, 1, 1, 2, 2);
  if (activation == 1) {
    ppocr::detail::kernels::PointwiseConvRelu(expected.data(), intermediate.data(),
                                              pw_w.data(), pw_b.data(), output_channels,
                                              channels, std::size_t(height) * width);
  } else if (activation == 2) {
    ppocr::detail::kernels::PointwiseConvHardSwish(expected.data(), intermediate.data(),
                                                   pw_w.data(), pw_b.data(), output_channels,
                                                   channels, std::size_t(height) * width);
  } else {
    ppocr::detail::kernels::PointwiseConv(expected.data(), intermediate.data(), pw_w.data(),
                                          pw_b.data(), output_channels, channels,
                                          std::size_t(height) * width);
    if (activation == 3) {
      ppocr::detail::kernels::Gelu(expected.data(), expected.data(), expected.size(),
                                   expected.size());
    }
  }
  const bool fused = ppocr::detail::kernels::DepthwisePointwiseConvFused(
      output.data(), input.data(), dw_w.data(), dw_b.data(), pw_w.data(), pw_b.data(),
      1, channels, output_channels, height, width, 5, 5, 1, 1, 2, 2, activation);
  if (!fused) {
    return true;
  }
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 5e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "fused 5x5 DW+PW mismatch C=" << channels << " M=" << output_channels
                << " H=" << height << " W=" << width << " act=" << activation
                << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyPointwiseDepthwise(int input_channels, int output_channels, int height,
                              int width, bool relu) {
  // Tiled 8-OC kernel is ENABLE-only after the 20-run miss versus two-node.
#if defined(_WIN32)
  _putenv("PPOCR_ENABLE_POINTWISE_DEPTHWISE=1");
#else
  setenv("PPOCR_ENABLE_POINTWISE_DEPTHWISE", "1", 1);
#endif
  const std::size_t in_n = std::size_t(input_channels) * height * width;
  const std::size_t out_n = std::size_t(output_channels) * height * width;
  std::vector<float> input(in_n), pw_w(std::size_t(output_channels) * input_channels);
  std::vector<float> pw_b(static_cast<std::size_t>(output_channels));
  std::vector<float> dw_w(std::size_t(output_channels) * 9);
  std::vector<float> dw_b(static_cast<std::size_t>(output_channels));
  std::vector<float> intermediate(out_n), expected(out_n), output(out_n, -99.F);
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < pw_w.size(); ++i) pw_w[i] = float(int(i % 31) - 15) * .015625F;
  for (std::size_t i = 0; i < pw_b.size(); ++i) pw_b[i] = float(int(i % 17) - 8) * .0625F;
  for (std::size_t i = 0; i < dw_w.size(); ++i) dw_w[i] = float(int(i % 23) - 11) * .03125F;
  for (std::size_t i = 0; i < dw_b.size(); ++i) dw_b[i] = float(int(i % 13) - 6) * .0625F;
  if (relu) {
    ppocr::detail::kernels::PointwiseConvRelu(
        intermediate.data(), input.data(), pw_w.data(), pw_b.data(), output_channels,
        input_channels, std::size_t(height) * width);
  } else {
    ppocr::detail::kernels::PointwiseConv(
        intermediate.data(), input.data(), pw_w.data(), pw_b.data(), output_channels,
        input_channels, std::size_t(height) * width);
  }
  ppocr::detail::kernels::DepthwiseConv(
      expected.data(), intermediate.data(), dw_w.data(), dw_b.data(), output_channels,
      height, width, height, width, 3, 3, 1, 1, 1, 1);
  const bool fused = ppocr::detail::kernels::PointwiseDepthwiseConvFused(
      output.data(), input.data(), pw_w.data(), pw_b.data(), dw_w.data(), dw_b.data(),
      1, input_channels, output_channels, height, width, relu);
  if (!fused) {
    if (height >= 32 && std::getenv("PPOCR_DISABLE_POINTWISE_DEPTHWISE") == nullptr) {
      std::cerr << "fused PW+DW did not run IC=" << input_channels
                << " OC=" << output_channels << " H=" << height << " W=" << width
                << " relu=" << relu << '\n';
      return false;
    }
    return true;
  }
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 5e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "fused PW+DW mismatch IC=" << input_channels
                << " OC=" << output_channels << " H=" << height << " W=" << width
                << " relu=" << relu << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyDepthwiseExpandGelu(int channels, int hidden, int height, int width) {
  // Opt-in C*W tiled kernel is ENABLE-only; force it so smoke drives the
  // shipped AVX-512 function rather than the two-pass fallback.
#if defined(_WIN32)
  _putenv("PPOCR_ENABLE_AVX512_DW_EXPAND_GELU=1");
#else
  setenv("PPOCR_ENABLE_AVX512_DW_EXPAND_GELU", "1", 1);
#endif
  const std::size_t n = std::size_t(channels) * height * width;
  std::vector<float> input(n), dw_w(std::size_t(channels) * 9);
  std::vector<float> dw_b(static_cast<std::size_t>(channels));
  std::vector<float> expand_w(std::size_t(hidden) * channels);
  std::vector<float> expand_b(static_cast<std::size_t>(hidden));
  std::vector<float> project_w(std::size_t(channels) * hidden);
  std::vector<float> project_b(static_cast<std::size_t>(channels));
  std::vector<float> intermediate(n), expected(n), output(n, -99.F);
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < dw_w.size(); ++i) dw_w[i] = float(int(i % 23) - 11) * .03125F;
  for (std::size_t i = 0; i < dw_b.size(); ++i) dw_b[i] = float(int(i % 13) - 6) * .0625F;
  for (std::size_t i = 0; i < expand_w.size(); ++i) expand_w[i] = float(int(i % 31) - 15) * .015625F;
  for (std::size_t i = 0; i < expand_b.size(); ++i) expand_b[i] = float(int(i % 17) - 8) * .0625F;
  for (std::size_t i = 0; i < project_w.size(); ++i) project_w[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < project_b.size(); ++i) project_b[i] = float(int(i % 11) - 5) * .0625F;
  ppocr::detail::kernels::DepthwiseConv(
      intermediate.data(), input.data(), dw_w.data(), dw_b.data(), channels, height,
      width, height, width, 3, 3, 1, 1, 1, 1);
  ppocr::detail::kernels::ExpandGeluProjectAdd(
      expected.data(), intermediate.data(), expand_w.data(), expand_b.data(),
      project_w.data(), project_b.data(), channels, hidden,
      std::size_t(height) * width);
  ppocr::detail::kernels::DepthwiseExpandGeluProjectAdd(
      output.data(), input.data(), dw_w.data(), dw_b.data(), expand_w.data(),
      expand_b.data(), project_w.data(), project_b.data(), 1, channels, hidden,
      height, width, 3, 3, 1, 1, 1, 1);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 8e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "fused DW+ExpandGelu mismatch C=" << channels << " hidden=" << hidden
                << " H=" << height << " W=" << width << " index=" << i
                << " got=" << output[i] << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyDepthwiseBatch(int batches, int channels, int input_height,
                          int input_width, int kernel, int stride, int pad) {
  const int output_height = (input_height + 2 * pad - kernel) / stride + 1;
  const int output_width = (input_width + 2 * pad - kernel) / stride + 1;
  const std::size_t input_batch = std::size_t(channels) * input_height * input_width;
  const std::size_t output_batch = std::size_t(channels) * output_height * output_width;
  std::vector<float> input(std::size_t(batches) * input_batch);
  std::vector<float> weights(std::size_t(channels) * kernel * kernel);
  std::vector<float> bias(static_cast<std::size_t>(channels));
  std::vector<float> expected(std::size_t(batches) * output_batch);
  std::vector<float> output(expected.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 23) - 11) * .03125F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 13) - 6) * .0625F;
  for (int batch = 0; batch < batches; ++batch) {
    ppocr::detail::kernels::DepthwiseConv(
        expected.data() + std::size_t(batch) * output_batch,
        input.data() + std::size_t(batch) * input_batch, weights.data(), bias.data(), channels,
        input_height, input_width, output_height, output_width, kernel, kernel, stride, stride,
        pad, pad);
  }
  ppocr::detail::kernels::DepthwiseConvBatch(
      output.data(), input.data(), weights.data(), bias.data(), batches, channels,
      input_height, input_width, output_height, output_width, kernel, kernel, stride, stride,
      pad, pad);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "batched depthwise mismatch N=" << batches << " C=" << channels
                << " H=" << input_height << " W=" << input_width << " K=" << kernel
                << " stride=" << stride << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyFoldedDepthwiseHardSwish(int channels, int height, int width, int kernel_h,
                                    int kernel_w, int pad_h, int pad_w) {
  const int out_h = height + 2 * pad_h - kernel_h + 1;
  const int out_w = width + 2 * pad_w - kernel_w + 1;
  if (out_h <= 0 || out_w <= 0) return false;
  const std::size_t in_n = std::size_t(channels) * height * width;
  const std::size_t out_n = std::size_t(channels) * out_h * out_w;
  std::vector<float> input(in_n), weights(std::size_t(channels) * kernel_h * kernel_w);
  std::vector<float> bias(static_cast<std::size_t>(channels));
  std::vector<float> factor(static_cast<std::size_t>(channels));
  std::vector<float> offset(static_cast<std::size_t>(channels));
  std::vector<float> expected(out_n), fused(out_n), folded_w(weights.size()), folded_b(bias.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 23) - 11) * .03125F;
  for (int i = 0; i < channels; ++i) {
    bias[static_cast<std::size_t>(i)] = float(int(i % 13) - 6) * .0625F;
    factor[static_cast<std::size_t>(i)] = 0.75F + float(i % 5) * .05F;
    offset[static_cast<std::size_t>(i)] = float(int(i % 7) - 3) * .03125F;
  }
  ppocr::detail::kernels::DepthwiseConv(expected.data(), input.data(), weights.data(),
                                        bias.data(), channels, height, width, out_h, out_w,
                                        kernel_h, kernel_w, 1, 1, pad_h, pad_w);
  ppocr::detail::kernels::BatchNormAffine(expected.data(), expected.data(), factor.data(),
                                          offset.data(), 1, channels, std::size_t(out_h) * out_w);
  ppocr::detail::kernels::HardSwish(expected.data(), expected.data(), expected.size());
  const int per = kernel_h * kernel_w;
  for (int c = 0; c < channels; ++c) {
    for (int k = 0; k < per; ++k)
      folded_w[std::size_t(c) * per + k] = weights[std::size_t(c) * per + k] * factor[c];
    folded_b[static_cast<std::size_t>(c)] =
        bias[static_cast<std::size_t>(c)] * factor[c] + offset[c];
  }
  ppocr::detail::kernels::DepthwiseConv(fused.data(), input.data(), folded_w.data(),
                                        folded_b.data(), channels, height, width, out_h, out_w,
                                        kernel_h, kernel_w, 1, 1, pad_h, pad_w);
  ppocr::detail::kernels::HardSwish(fused.data(), fused.data(), fused.size());
  for (std::size_t i = 0; i < expected.size(); ++i) {
    const float tolerance = 4e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(fused[i] - expected[i]) > tolerance) {
      std::cerr << "folded dw+hswish mismatch C=" << channels << " " << height << "x" << width
                << " k=" << kernel_h << "x" << kernel_w << " index=" << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyDepthwiseScalar(int channels, int height, int width, int kernel, int pad) {
  const int output_height = height;
  const int output_width = width;
  const std::size_t plane = std::size_t(height) * width;
  std::vector<float> input(std::size_t(channels) * plane);
  std::vector<float> weights(std::size_t(channels) * kernel * kernel);
  std::vector<float> bias(static_cast<std::size_t>(channels));
  std::vector<float> expected(input.size()), output(input.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 23) - 11) * .03125F;
  for (int i = 0; i < channels; ++i) bias[static_cast<std::size_t>(i)] = float(int(i % 13) - 6) * .0625F;
  for (int channel = 0; channel < channels; ++channel) {
    const float* in = input.data() + std::size_t(channel) * plane;
    const float* filter = weights.data() + std::size_t(channel) * kernel * kernel;
    float* out = expected.data() + std::size_t(channel) * plane;
    const float base = bias[static_cast<std::size_t>(channel)];
    for (int y = 0; y < output_height; ++y) {
      for (int x = 0; x < output_width; ++x) {
        float sum = base;
        for (int ky = 0; ky < kernel; ++ky) {
          const int iy = y + ky - pad;
          if (iy < 0 || iy >= height) continue;
          for (int kx = 0; kx < kernel; ++kx) {
            const int ix = x + kx - pad;
            if (ix >= 0 && ix < width)
              sum += in[std::size_t(iy) * width + ix] * filter[ky * kernel + kx];
          }
        }
        out[std::size_t(y) * output_width + x] = sum;
      }
    }
  }
  ppocr::detail::kernels::DepthwiseConv(output.data(), input.data(), weights.data(),
                                        bias.data(), channels, height, width, output_height,
                                        output_width, kernel, kernel, 1, 1, pad, pad);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "depthwise scalar mismatch C=" << channels << " H=" << height
                << " W=" << width << " K=" << kernel << " index=" << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConvBatch(int batches, int input_channels, int output_channels,
                     int input_height, int input_width, int kernel_height,
                     int kernel_width, int stride_height, int stride_width,
                     int pad_top, int pad_left, bool relu) {
  const int output_height = (input_height + 2 * pad_top - kernel_height) / stride_height + 1;
  const int output_width = (input_width + 2 * pad_left - kernel_width) / stride_width + 1;
  const std::size_t input_batch = std::size_t(input_channels) * input_height * input_width;
  const std::size_t output_batch = std::size_t(output_channels) * output_height * output_width;
  std::vector<float> input(std::size_t(batches) * input_batch);
  std::vector<float> weights(std::size_t(output_channels) * input_channels * kernel_height * kernel_width);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> expected(std::size_t(batches) * output_batch);
  std::vector<float> output(expected.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 43) - 21) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 37) - 18) * .015625F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 17) - 8) * .0625F;
  for (int batch = 0; batch < batches; ++batch) {
    ppocr::detail::kernels::Conv2d(
        expected.data() + std::size_t(batch) * output_batch,
        input.data() + std::size_t(batch) * input_batch, weights.data(), bias.data(),
        input_channels, output_channels, input_height, input_width, output_height,
        output_width, kernel_height, kernel_width, stride_height, stride_width,
        pad_top, pad_left, relu);
  }
  ppocr::detail::kernels::Conv2dBatch(
      output.data(), input.data(), weights.data(), bias.data(), batches, input_channels,
      output_channels, input_height, input_width, output_height, output_width,
      kernel_height, kernel_width, stride_height, stride_width, pad_top, pad_left, relu);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "batched 3x3 mismatch N=" << batches << " C=" << input_channels
                << " M=" << output_channels << " H=" << input_height << " W="
                << input_width << " K=" << kernel_height << 'x' << kernel_width
                << " stride=" << stride_height << 'x' << stride_width << " relu=" << relu
                << " index=" << i << " got=" << output[i] << " expected=" << expected[i]
                << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyPointwiseHardSwishBatch(int batches, int input_channels,
                                   int output_channels, std::size_t plane) {
  const std::size_t input_batch = std::size_t(input_channels) * plane;
  const std::size_t output_batch = std::size_t(output_channels) * plane;
  std::vector<float> input(std::size_t(batches) * input_batch);
  std::vector<float> weights(std::size_t(output_channels) * input_channels);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> expected(std::size_t(batches) * output_batch);
  std::vector<float> output(expected.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 47) - 23) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 29) - 14) * .03125F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 11) - 5) * .125F;
  for (int batch = 0; batch < batches; ++batch) {
    ppocr::detail::kernels::PointwiseConvHardSwish(
        expected.data() + std::size_t(batch) * output_batch,
        input.data() + std::size_t(batch) * input_batch, weights.data(), bias.data(),
        output_channels, input_channels, plane);
  }
  ppocr::detail::kernels::PointwiseConvHardSwishBatch(
      output.data(), input.data(), weights.data(), bias.data(), batches,
      output_channels, input_channels, plane);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "batched pointwise hardswish mismatch N=" << batches << " C="
                << input_channels << " M=" << output_channels << " plane=" << plane
                << " index=" << i << " got=" << output[i] << " expected=" << expected[i]
                << '\n';
      return false;
    }
  }
  return true;
}

bool VerifySqueezeExcitation(int batches, int channels, int reduced, int height,
                             int width, bool residual) {
  const std::size_t spatial = std::size_t(height) * width;
  const std::size_t count = std::size_t(batches) * channels * spatial;
  std::vector<float> values(count), first_w(std::size_t(reduced) * channels);
  std::vector<float> first_b(static_cast<std::size_t>(reduced));
  std::vector<float> second_w(std::size_t(channels) * reduced);
  std::vector<float> second_b(static_cast<std::size_t>(channels));
  for (std::size_t i = 0; i < values.size(); ++i) values[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < first_w.size(); ++i) first_w[i] = float(int(i % 23) - 11) * .015625F;
  for (std::size_t i = 0; i < first_b.size(); ++i) first_b[i] = float(int(i % 13) - 6) * .0625F;
  for (std::size_t i = 0; i < second_w.size(); ++i) second_w[i] = float(int(i % 29) - 14) * .015625F;
  for (std::size_t i = 0; i < second_b.size(); ++i) second_b[i] = float(int(i % 11) - 5) * .0625F;
  std::vector<float> expected = values;
  constexpr float alpha = .2F, beta = .5F;
  for (int batch = 0; batch < batches; ++batch) {
    std::vector<float> means(static_cast<std::size_t>(channels), 0.F);
    for (int channel = 0; channel < channels; ++channel) {
      const float* plane = expected.data() +
          (std::size_t(batch) * channels + channel) * spatial;
      float sum = 0.F;
      for (std::size_t i = 0; i < spatial; ++i) sum += plane[i];
      means[static_cast<std::size_t>(channel)] = sum / float(spatial);
    }
    std::vector<float> hidden(static_cast<std::size_t>(reduced));
    for (int output = 0; output < reduced; ++output) {
      float sum = first_b[static_cast<std::size_t>(output)];
      const float* weights = first_w.data() + std::size_t(output) * channels;
      for (int input_channel = 0; input_channel < channels; ++input_channel)
        sum += weights[input_channel] * means[static_cast<std::size_t>(input_channel)];
      hidden[static_cast<std::size_t>(output)] = std::max(0.F, sum);
    }
    for (int output = 0; output < channels; ++output) {
      float sum = second_b[static_cast<std::size_t>(output)];
      const float* weights = second_w.data() + std::size_t(output) * reduced;
      for (int hidden_channel = 0; hidden_channel < reduced; ++hidden_channel)
        sum += weights[hidden_channel] * hidden[static_cast<std::size_t>(hidden_channel)];
      const float gate = std::clamp(alpha * sum + beta, 0.F, 1.F);
      const float scale = residual ? (1.F + gate) : gate;
      float* plane = expected.data() +
          (std::size_t(batch) * channels + output) * spatial;
      for (std::size_t i = 0; i < spatial; ++i) plane[i] *= scale;
    }
  }
  auto output = values;
  ppocr::detail::kernels::SqueezeExcitationGateInplace(
      output.data(), first_w.data(), first_b.data(), second_w.data(), second_b.data(),
      batches, channels, reduced, spatial, alpha, beta, residual);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 5e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "SE gate mismatch residual=" << residual << " C=" << channels
                << " R=" << reduced << " index=" << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConvTransposeScalar(int input_channels, int output_channels, int input_height,
                               int input_width) {
  const int output_height = input_height * 2;
  const int output_width = input_width * 2;
  const std::size_t input_plane = std::size_t(input_channels) * input_height * input_width;
  const std::size_t output_plane = std::size_t(output_channels) * output_height * output_width;
  std::vector<float> input(input_plane), weights(std::size_t(input_channels) * output_channels * 4);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> expected(output_plane), output(output_plane);
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 23) - 11) * .03125F;
  for (int i = 0; i < output_channels; ++i) bias[static_cast<std::size_t>(i)] = float(int(i % 13) - 6) * .0625F;
  const std::size_t in_hw = std::size_t(input_height) * input_width;
  const std::size_t out_hw = std::size_t(output_height) * output_width;
  for (int output_channel = 0; output_channel < output_channels; ++output_channel) {
    float* out = expected.data() + std::size_t(output_channel) * out_hw;
    std::fill_n(out, out_hw, bias[static_cast<std::size_t>(output_channel)]);
    for (int input_channel = 0; input_channel < input_channels; ++input_channel) {
      const float* src = input.data() + std::size_t(input_channel) * in_hw;
      const float* w = weights.data() + (std::size_t(input_channel) * output_channels +
                                         output_channel) * 4;
      for (int y = 0; y < input_height; ++y) {
        const float* row = src + std::size_t(y) * input_width;
        float* dst0 = out + std::size_t(2 * y) * output_width;
        float* dst1 = dst0 + output_width;
        for (int x = 0; x < input_width; ++x) {
          const float value = row[x];
          const int xx = 2 * x;
          dst0[xx] += value * w[0]; dst0[xx + 1] += value * w[1];
          dst1[xx] += value * w[2]; dst1[xx + 1] += value * w[3];
        }
      }
    }
  }
  ppocr::detail::kernels::ConvTranspose2x2(output.data(), input.data(), weights.data(),
                                           bias.data(), input_channels, output_channels,
                                           input_height, input_width);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "transpose2x2 scalar mismatch C=" << input_channels << " M=" << output_channels
                << " H=" << input_height << " W=" << input_width << " index=" << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConvTransposeAct(int input_channels, int output_channels, int input_height,
                            int input_width, int act) {
  const int output_height = input_height * 2;
  const int output_width = input_width * 2;
  const std::size_t input_plane = std::size_t(input_channels) * input_height * input_width;
  const std::size_t output_count =
      std::size_t(output_channels) * output_height * output_width;
  std::vector<float> input(input_plane), weights(std::size_t(input_channels) * output_channels * 4);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> expected(output_count), output(output_count);
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 23) - 11) * .03125F;
  for (int i = 0; i < output_channels; ++i)
    bias[static_cast<std::size_t>(i)] = float(int(i % 13) - 6) * .0625F;
  ppocr::detail::kernels::ConvTranspose2x2(expected.data(), input.data(), weights.data(),
                                           bias.data(), input_channels, output_channels,
                                           input_height, input_width, 0);
  if (act == 1) {
    for (float& value : expected) value = std::max(value, 0.F);
  } else if (act == 2) {
    for (float& value : expected) value = 1.F / (1.F + std::exp(-value));
  }
  ppocr::detail::kernels::ConvTranspose2x2(output.data(), input.data(), weights.data(),
                                           bias.data(), input_channels, output_channels,
                                           input_height, input_width, act);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = (act == 2 ? 3e-5F : 3e-5F) * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "transpose2x2 act=" << act << " mismatch C=" << input_channels
                << " M=" << output_channels << " H=" << input_height << " W=" << input_width
                << " index=" << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConvTransposeChain(int input_channels, int mid_channels, int output_channels,
                              int input_height, int input_width) {
  const int mid_h = input_height * 2, mid_w = input_width * 2;
  const int out_h = input_height * 4, out_w = input_width * 4;
  const std::size_t in_n = std::size_t(input_channels) * input_height * input_width;
  const std::size_t mid_n = std::size_t(mid_channels) * mid_h * mid_w;
  const std::size_t out_n = std::size_t(output_channels) * out_h * out_w;
  std::vector<float> input(in_n), w0(std::size_t(input_channels) * mid_channels * 4);
  std::vector<float> b0(static_cast<std::size_t>(mid_channels));
  std::vector<float> w1(std::size_t(mid_channels) * output_channels * 4);
  std::vector<float> b1(static_cast<std::size_t>(output_channels));
  std::vector<float> mid(mid_n), expected(out_n), output(out_n);
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < w0.size(); ++i) w0[i] = float(int(i % 23) - 11) * .03125F;
  for (int i = 0; i < mid_channels; ++i) b0[static_cast<std::size_t>(i)] = float(int(i % 13) - 6) * .0625F;
  for (std::size_t i = 0; i < w1.size(); ++i) w1[i] = float(int(i % 19) - 9) * .03125F;
  for (int i = 0; i < output_channels; ++i) b1[static_cast<std::size_t>(i)] = float(int(i % 11) - 5) * .0625F;
  ppocr::detail::kernels::ConvTranspose2x2(mid.data(), input.data(), w0.data(), b0.data(),
                                           input_channels, mid_channels, input_height,
                                           input_width, 1);
  ppocr::detail::kernels::ConvTranspose2x2(expected.data(), mid.data(), w1.data(), b1.data(),
                                           mid_channels, output_channels, mid_h, mid_w, 2);
  ppocr::detail::kernels::ConvTranspose2x2Chain(output.data(), input.data(), w0.data(), b0.data(),
                                                w1.data(), b1.data(), input_channels, mid_channels,
                                                output_channels, input_height, input_width);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 4e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "transpose2x2 chain mismatch C=" << input_channels << " M=" << mid_channels
                << " O=" << output_channels << " H=" << input_height << " W=" << input_width
                << " index=" << i << " got=" << output[i] << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyConvTransposeBatch(int batches, int input_channels, int output_channels,
                              int input_height, int input_width) {
  const std::size_t input_batch = std::size_t(input_channels) * input_height * input_width;
  const std::size_t output_batch = std::size_t(output_channels) * (input_height * 2) * (input_width * 2);
  std::vector<float> input(std::size_t(batches) * input_batch);
  std::vector<float> weights(std::size_t(input_channels) * output_channels * 4);
  std::vector<float> bias(static_cast<std::size_t>(output_channels));
  std::vector<float> expected(std::size_t(batches) * output_batch);
  std::vector<float> output(expected.size());
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 41) - 20) * .03125F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 23) - 11) * .03125F;
  for (std::size_t i = 0; i < bias.size(); ++i) bias[i] = float(int(i % 13) - 6) * .0625F;
  for (int batch = 0; batch < batches; ++batch) {
    ppocr::detail::kernels::ConvTranspose2x2(
        expected.data() + std::size_t(batch) * output_batch,
        input.data() + std::size_t(batch) * input_batch, weights.data(), bias.data(),
        input_channels, output_channels, input_height, input_width);
  }
  ppocr::detail::kernels::ConvTranspose2x2Batch(
      output.data(), input.data(), weights.data(), bias.data(), batches, input_channels,
      output_channels, input_height, input_width);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-5F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "batched transpose2x2 mismatch N=" << batches << " C=" << input_channels
                << " M=" << output_channels << " H=" << input_height << " W=" << input_width
                << " index=" << i << " got=" << output[i] << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyBatchNormAffine(int batches, int channels, std::size_t spatial) {
  const std::size_t count = std::size_t(batches) * channels * spatial;
  std::vector<float> input(count), scale(channels), shift(channels), expected(count), output(count);
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 53) - 26) * .03125F;
  for (int channel = 0; channel < channels; ++channel) {
    scale[channel] = float((channel % 11) + 1) * .09375F;
    shift[channel] = float((channel % 7) - 3) * .125F;
  }
  for (int batch = 0; batch < batches; ++batch) for (int channel = 0; channel < channels; ++channel) {
    const auto offset = (std::size_t(batch) * channels + channel) * spatial;
    for (std::size_t index = 0; index < spatial; ++index) {
      expected[offset + index] = input[offset + index] * scale[channel] + shift[channel];
    }
  }
  ppocr::detail::kernels::BatchNormAffine(output.data(), input.data(), scale.data(), shift.data(),
                                           batches, channels, spatial);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-6F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "BatchNormAffine output mismatch at " << i << '\n';
      return false;
    }
  }
  auto inplace = input;
  ppocr::detail::kernels::BatchNormAffine(inplace.data(), inplace.data(), scale.data(), shift.data(),
                                           batches, channels, spatial);
  for (std::size_t i = 0; i < inplace.size(); ++i) {
    if (inplace[i] != expected[i]) {
      std::cerr << "BatchNormAffine inplace mismatch at " << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyNearestResize2xAdd(int batches, int channels, int height, int width) {
  const int output_width = width * 2;
  const std::size_t input_count = std::size_t(batches) * channels * height * width;
  const std::size_t output_count = input_count * 4;
  std::vector<float> input(input_count), residual(output_count), expected(output_count), output(output_count);
  for (std::size_t i = 0; i < input.size(); ++i) input[i] = float(int(i % 59) - 29) * .03125F;
  for (std::size_t i = 0; i < residual.size(); ++i) residual[i] = float(int(i % 43) - 21) * .015625F;
  for (int plane = 0; plane < batches * channels; ++plane) {
    const float* src = input.data() + std::size_t(plane) * height * width;
    const float* add = residual.data() + std::size_t(plane) * height * width * 4;
    float* dst = expected.data() + std::size_t(plane) * height * width * 4;
    for (int y = 0; y < height; ++y) for (int x = 0; x < width; ++x) {
      const float value = src[std::size_t(y) * width + x];
      const auto offset = std::size_t(y * 2) * output_width + std::size_t(x) * 2;
      dst[offset] = value + add[offset]; dst[offset + 1] = value + add[offset + 1];
      dst[offset + output_width] = value + add[offset + output_width];
      dst[offset + output_width + 1] = value + add[offset + output_width + 1];
    }
  }
  ppocr::detail::kernels::NearestResize2xAdd(output.data(), input.data(), residual.data(),
                                               batches, channels, height, width);
  for (std::size_t i = 0; i < output.size(); ++i) {
    if (output[i] != expected[i]) {
      std::cerr << "nearest resize-add mismatch N=" << batches << " C=" << channels
                << " H=" << height << " W=" << width << " index=" << i << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyAveragePool3x2Valid(int planes, int height, int width) {
  const int output_height = (height - 3) / 3 + 1;
  const int output_width = (width - 2) / 2 + 1;
  const std::size_t input_plane = std::size_t(height) * width;
  const std::size_t output_plane = std::size_t(output_height) * output_width;
  std::vector<float> input(std::size_t(planes) * input_plane);
  std::vector<float> expected(std::size_t(planes) * output_plane), output(expected.size());
  for (std::size_t i = 0; i < input.size(); ++i) {
    input[i] = float(int(i % 37) - 18) * .03125F;
  }
  for (int plane = 0; plane < planes; ++plane) {
    const float* source = input.data() + std::size_t(plane) * input_plane;
    float* destination = expected.data() + std::size_t(plane) * output_plane;
    for (int y = 0; y < output_height; ++y) for (int x = 0; x < output_width; ++x) {
      float sum{};
      for (int ky = 0; ky < 3; ++ky) for (int kx = 0; kx < 2; ++kx) {
        sum += source[std::size_t(y * 3 + ky) * width + x * 2 + kx];
      }
      destination[std::size_t(y) * output_width + x] = sum / 6.F;
    }
  }
  ppocr::detail::kernels::AveragePool3x2Valid(output.data(), input.data(), planes, height, width);
  for (std::size_t i = 0; i < output.size(); ++i) {
    const float tolerance = 3e-6F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance) {
      std::cerr << "average pool 3x2 valid mismatch at " << i << " got=" << output[i]
                << " expected=" << expected[i] << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyCtcTop1(int batches, int steps, int vocab) {
  const std::size_t rows = std::size_t(batches) * steps;
  std::vector<float> logits(rows * vocab);
  std::vector<int> expected_indices(rows), output_indices(rows);
  std::vector<float> expected_probabilities(rows), output_probabilities(rows);
  for (std::size_t row = 0; row < rows; ++row) {
    for (int column = 0; column < vocab; ++column) {
      logits[row * vocab + column] = float(int((row * 31 + std::size_t(column) * 17) % 127) - 63) * .03125F;
    }
    // Cover blank and repeated top-1 decisions, including vector tails.
    logits[row * vocab + (row % 5 == 0 ? 0 : int((row + 7) % vocab))] = 3.F + float(row % 3);
  }
  ppocr::detail::kernels::CtcTop1Scalar(expected_indices.data(), expected_probabilities.data(),
                                         logits.data(), rows, steps, vocab);
  ppocr::detail::kernels::CtcTop1(output_indices.data(), output_probabilities.data(),
                                   logits.data(), rows, steps, vocab);
  for (std::size_t row = 0; row < rows; ++row) {
    if (output_indices[row] != expected_indices[row]) {
      std::cerr << "CTC Top1 index mismatch row=" << row << '\n';
      return false;
    }
    const float tolerance = 3e-6F * std::max(1.F, std::abs(expected_probabilities[row]));
    if (std::abs(output_probabilities[row] - expected_probabilities[row]) > tolerance) {
      std::cerr << "CTC Top1 probability mismatch row=" << row << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyCtcTop1FirstMaximum() {
  // SIMD ArgMax must retain the scalar executor's first-maximum rule across
  // vector lanes and tails. Use a 6,906-way row so this covers the production
  // Latin vocabulary geometry as well as a small vector-tail row.
  for (const int vocab : {31, 6906}) {
    const int batches = 1, steps = 4;
    std::vector<float> logits(std::size_t(batches) * steps * vocab, -2.F);
    for (int step = 0; step < steps; ++step) {
      float* row = logits.data() + std::size_t(step) * vocab;
      row[0] = 1.F;
      row[std::min(vocab - 1, 3)] = 5.F;
      row[std::min(vocab - 1, 16)] = 5.F;  // tie: first maximum must win.
      row[vocab - 1] = step == 1 ? 6.F : 5.F;
    }
    std::vector<int> expected_indices(steps), output_indices(steps);
    std::vector<float> expected_probabilities(steps), output_probabilities(steps);
    ppocr::detail::kernels::CtcTop1Scalar(expected_indices.data(), expected_probabilities.data(),
                                           logits.data(), steps, steps, vocab);
    ppocr::detail::kernels::CtcTop1(output_indices.data(), output_probabilities.data(),
                                     logits.data(), steps, steps, vocab);
    for (int row = 0; row < steps; ++row) {
      if (output_indices[row] != expected_indices[row] ||
          std::abs(output_probabilities[row] - expected_probabilities[row]) > 1e-5F) {
        std::cerr << "CTC Top1 first-maximum mismatch vocab=" << vocab << " row=" << row
                  << " expected_index=" << expected_indices[row]
                  << " output_index=" << output_indices[row]
                  << " expected_probability=" << expected_probabilities[row]
                  << " output_probability=" << output_probabilities[row] << '\n';
        return false;
      }
    }
  }
  return true;
}

bool VerifyGemmCtcTop1(int batches, int steps, int vocab, int depth) {
  const std::size_t rows = std::size_t(batches) * steps;
  std::vector<float> left(rows * depth), weights(std::size_t(depth) * vocab), bias(vocab), logits(rows * vocab);
  std::vector<int> expected_indices(rows), output_indices(rows);
  std::vector<float> expected_probabilities(rows), output_probabilities(rows);
  for (std::size_t i = 0; i < left.size(); ++i) left[i] = float(int(i % 37) - 18) * .0625F;
  for (std::size_t i = 0; i < weights.size(); ++i) weights[i] = float(int(i % 41) - 20) * .03125F;
  for (int i = 0; i < vocab; ++i) bias[std::size_t(i)] = float(i % 11) * .0625F - .25F;
  ppocr::detail::kernels::Gemm(logits.data(), left.data(), weights.data(), bias.data(),
                                int(rows), vocab, depth);
  ppocr::detail::kernels::CtcTop1Scalar(expected_indices.data(), expected_probabilities.data(),
                                         logits.data(), rows, steps, vocab);
  ppocr::detail::kernels::GemmCtcTop1(output_indices.data(), output_probabilities.data(),
                                       left.data(), weights.data(), bias.data(), int(rows), depth,
                                       vocab, steps);
  for (std::size_t row = 0; row < rows; ++row) {
    if (output_indices[row] != expected_indices[row]) {
      std::cerr << "GEMM CTC Top1 index mismatch row=" << row << '\n';
      return false;
    }
    const float tolerance = 3e-6F * std::max(1.F, std::abs(expected_probabilities[row]));
    if (std::abs(output_probabilities[row] - expected_probabilities[row]) > tolerance) {
      std::cerr << "GEMM CTC Top1 probability mismatch row=" << row << '\n';
      return false;
    }
  }
  return true;
}

bool VerifyApproximateGelu() {
  // The optional x86 path is intentionally a Padé-tanh GELU approximation.
  // Validate both in-place and separate buffers against its scalar formula so
  // AVX-512, AVX2 and scalar tails share identical deployment semantics.
  // Cross the model-specific approximate-GELU threshold and retain a vector
  // tail, covering the runtime dispatch used by broad medium feature maps.
  std::vector<float> input(262144 + 13), output(input.size()), inplace(input.size());
  for (std::size_t i = 0; i < input.size(); ++i) {
    input[i] = float(int(i % 127) - 63) * .0625F;
    inplace[i] = input[i];
  }
  constexpr float c0 = .7978845608028654F, c1 = .044715F;
  std::vector<float> expected(input.size());
  for (std::size_t i = 0; i < input.size(); ++i) {
    const float x = input[i];
    const float z = c0 * (x + c1 * x * x * x);
    const float z2 = z * z;
    const float t = std::clamp(z * (27.F + z2) / (27.F + 9.F * z2), -1.F, 1.F);
    expected[i] = .5F * x * (1.F + t);
  }
  ppocr::detail::kernels::Gelu(output.data(), input.data(), input.size(), input.size());
  ppocr::detail::kernels::Gelu(inplace.data(), inplace.data(), inplace.size(), inplace.size());
  for (std::size_t i = 0; i < input.size(); ++i) {
    const float tolerance = 3e-6F * std::max(1.F, std::abs(expected[i]));
    if (std::abs(output[i] - expected[i]) > tolerance ||
        std::abs(inplace[i] - expected[i]) > tolerance) {
      std::cerr << "approximate GELU mismatch index=" << i << '\n';
      return false;
    }
  }
  return true;
}

}  // namespace

int main() {
  // Drive ENABLE-only MaxPoolConcat/CTC kernels from the shipped functions.
#if defined(_WIN32)
  _putenv("PPOCR_ENABLE_AVX512_MAXPOOL_TINY_PARALLEL=1");
  _putenv("PPOCR_ENABLE_AVX512_CONCAT_S2_ROW_PF=1");
  _putenv("PPOCR_ENABLE_AVX512_GEMM16NARROW=1");
  _putenv("PPOCR_ENABLE_AVX512_POINTWISE16_TINY=1");
  _putenv("PPOCR_ENABLE_CTC_VOCAB_BLOCK=1");
  _putenv("PPOCR_ENABLE_AVX512_CONCAT_ONEPASS=1");
  _putenv("PPOCR_ENABLE_AVX512_TRANSPOSE2X2_TILE4=1");
  _putenv("PPOCR_ENABLE_AVX512_RESIZE2X_PF=1");
  _putenv("PPOCR_ENABLE_POINTWISE_DEPTHWISE=1");
  _putenv("PPOCR_FUSED_TERMINAL_CTC_TILE_ROWS=16");
#else
  setenv("PPOCR_ENABLE_AVX512_MAXPOOL_TINY_PARALLEL", "1", 1);
  setenv("PPOCR_ENABLE_AVX512_CONCAT_S2_ROW_PF", "1", 1);
  setenv("PPOCR_ENABLE_AVX512_GEMM16NARROW", "1", 1);
  setenv("PPOCR_ENABLE_AVX512_POINTWISE16_TINY", "1", 1);
  setenv("PPOCR_ENABLE_CTC_VOCAB_BLOCK", "1", 1);
  setenv("PPOCR_ENABLE_AVX512_CONCAT_ONEPASS", "1", 1);
  setenv("PPOCR_ENABLE_AVX512_TRANSPOSE2X2_TILE4", "1", 1);
  setenv("PPOCR_ENABLE_AVX512_RESIZE2X_PF", "1", 1);
  setenv("PPOCR_ENABLE_POINTWISE_DEPTHWISE", "1", 1);
  setenv("PPOCR_FUSED_TERMINAL_CTC_TILE_ROWS", "16", 1);
#endif
  // Odd row counts exercise the paired-row SIMD loop plus its scalar/SIMD
  // tail; odd/vector-boundary column counts cover both vector and scalar
  // write paths. The last case is recognizer-projection sized.
  if (!VerifyRgbByteWiden() || !VerifyIdentityRgbToNchw() ||
      !VerifyBilinearRgbToNchw(720, 152, 0, 0, 720, 152, 704, 160, 704, true) ||
      !VerifyBilinearRgbToNchw(64, 32, 2, 1, 37, 19, 41, 17, 48, false) ||
      !VerifyBilinearRgbToNchw(8, 8, 0, 0, 1, 1, 5, 5, 8, false) ||
      !VerifyBilinearRgbToNchw(32, 16, 0, 0, 32, 16, 32, 16, 32, true) ||
      !VerifyExactGelu() ||
      !VerifySigmoid(17) || !VerifySigmoid(262144 + 13) ||
      !VerifyMaxPool2x2Valid(16, 41, 39) || !VerifyMaxPool2x2Valid(3, 17, 19) ||
      !VerifyMaxPool2x2Valid(16, 64, 63) || !VerifyMaxPool2x2Valid(8, 128, 127) ||
      !VerifyMaxPool2x2Same(16, 41, 39) || !VerifyMaxPool2x2Same(8, 64, 63) ||
      !VerifyMaxPool2x2Same(3, 17, 19) || !VerifyMaxPool2x2Same(16, 80, 352)) return 1;
  if (!VerifyConv3x3Stride2(3, 16, 41, 39, true) ||
      !VerifyConv3x3Stride2(3, 16, 65, 63, true) ||
      !VerifyConv3x3Stride2(3, 16, 160, 128, true) ||
      !VerifyConv3x3Stride2(3, 5, 37, 35, true) ||
      !VerifyConv3x3Stride2(11, 9, 39, 41, false) ||
      !VerifyConv3x3Stride2(24, 48, 41, 39, true) ||
      !VerifyConv3x3Stride2(32, 16, 41, 39, true)) return 1;
  if (!VerifyConcatChannelConv(4, 16, 16, 41, 39, true) ||
      !VerifyConcatChannelConv(4, 16, 16, 40, 176, true) ||
      !VerifyConcatChannelConv(4, 16, 16, 40, 176, false) ||
      !VerifyConcatChannelConv(3, 5, 8, 17, 19, false) ||
      !VerifyConcatChannelConv(2, 7, 9, 33, 31, true) ||
      !VerifyConcatChannelConv(3, 8, 16, 41, 39, true, 2, 2) ||
      !VerifyConcatChannelConv(2, 16, 16, 80, 352, true, 2, 2) ||
      !VerifyConcatChannelConv(2, 7, 9, 33, 31, false, 2, 2)) return 1;
  {
    const int stem_channels[] = {16, 8, 16};
    if (!VerifyConcatChannelConvMixed(stem_channels, 3, 16, 41, 39, true, 2, 2) ||
        !VerifyConcatChannelConvMixed(stem_channels, 3, 16, 37, 35, false, 2, 2)) {
      return 1;
    }
  }
  if (!VerifyConcatConvDualTranspose(4, 16, 16, 1, 9, 11) ||
      !VerifyConcatConvDualTranspose(4, 16, 16, 1, 40, 176)) return 1;
  if (!VerifyLayerNorm(3, 17) || !VerifyLayerNorm(5, 192) ||
      !VerifyLayerNorm(4, 193)) return 1;
  for (const auto [rows, cols, depth] : {
           std::array<int, 3>{1, 1, 1},
           std::array<int, 3>{2, 7, 5},
           std::array<int, 3>{3, 17, 13},
           std::array<int, 3>{5, 33, 31},
           std::array<int, 3>{7, 65, 47},
           std::array<int, 3>{8, 6913, 192},
           // Terminal CTC projections for the shipped recognizers.  These
           // wide, four-row cases exercise the AVX-512 four-row tile and its
           // lower-precision-free vector/tail handling without requiring an
           // OCR model load in this low-level smoke binary.
           std::array<int, 3>{4, 6906, 80},
           std::array<int, 3>{16, 6906, 80},
           std::array<int, 3>{4, 18710, 120},
           std::array<int, 3>{4, 18710, 192},
       }) {
    if (!VerifyGemm(rows, cols, depth, false) || !VerifyGemm(rows, cols, depth, true)) return 1;
  }
  // A large square map takes the AVX-512 eight-output Conv tile, while the
  // odd output count exercises its four-channel fallback/tail. ReLU covers
  // the detector's fused writeback path.
  if (!VerifyConv3x3Tile(32, 8, 128, 128, false) ||
      !VerifyConv3x3Tile(32, 9, 128, 128, true) ||
      !VerifyConv3x3Tile(64, 16, 65, 63, true)) return 1;
  if (!VerifyDetStemFromNchw(41, 39) || !VerifyDetStemFromNchw(160, 128) ||
      !VerifyDetStemFromNchw(64, 64)) return 1;
  if (!VerifyConv2x2SameUpper(32, 8, 65, 63, false) ||
      !VerifyConv2x2SameUpper(32, 9, 64, 64, true) ||
      !VerifyConv2x2SameUpper(16, 8, 41, 39, true) ||
      !VerifyConv2x2SameUpper(16, 8, 80, 352, true) ||
      !VerifyConv2x2SameUpper(8, 16, 41, 39, true) ||
      !VerifyConv2x2SameUpper(8, 16, 80, 352, true) ||
      !VerifyConv2x2SameUpper(8, 12, 17, 15, false)) return 1;
  // PP-OCRv6 medium uses 9x9 depthwise MobileNet context filters before a
  // pointwise projection.  Exercise a non-square plane and a tail channel so
  // AVX-512, AVX2, NEON, and scalar dispatches all cover the same geometry.
  if (!VerifyDepthwiseBatch(2, 257, 37, 41, 9, 1, 4)) return 1;
  // The opt-in wide tile has an eight-channel SIMD body and a four-channel
  // tail. These padded 5x5/7x7 cases cover the medium-detector context path.
  if (!VerifyConvWideTile(32, 8, 64, 63, 5) ||
      !VerifyConvWideTile(32, 9, 64, 63, 7)) return 1;
  // Exercise the NCHW cross-crop schedule, including non-SIMD tails and a
  // work size that crosses the persistent-pool threshold.
  if (!VerifyPointwiseConv(48, 96, 1284) || !VerifyPointwiseConv(96, 48, 1284) ||
      !VerifyPointwiseConv(48, 97, 1000) || !VerifyPointwiseConv(11, 9, 37) ||
      !VerifyPointwiseConv(16, 16, 7040) || !VerifyPointwiseConv(16, 24, 2112) ||
      !VerifyPointwiseConv(32, 64, 7040) || !VerifyPointwiseConv(32, 64, 272) ||
      !VerifyPointwiseConv(64, 16, 7040) || !VerifyPointwiseConv(64, 16, 272) ||
      !VerifyPointwiseConv(16, 32, 7040) || !VerifyPointwiseConv(16, 32, 80) ||
      !VerifyPointwiseConvRelu(16, 32, 7040) || !VerifyPointwiseConvRelu(16, 32, 80) ||
      !VerifyPointwiseConvRelu(32, 64, 7040) || !VerifyPointwiseConvRelu(7, 9, 37)) {
    return 1;
  }
  if (!VerifyExpandGeluProjectAdd(48, 96, 1284) ||
      !VerifyExpandGeluProjectAdd(48, 96, 1000) ||
      !VerifyExpandGeluProjectAdd(11, 9, 37) ||
      !VerifyExpandGeluProjectAdd(8, 16, 16) ||
      !VerifyExpandGeluProjectAdd(4, 5, 17)) {
    return 1;
  }
  for (int activation = 0; activation < 3; ++activation) {
    if (!VerifyPointwiseResidualBatch(3, 7, 9, 37, activation)) return 1;
  }
  if (!VerifyPointwiseResidualBatch(3, 64, 64, 1024, 0)) return 1;
  if (!VerifyDepthwiseBatch(3, 5, 17, 19, 3, 1, 1) ||
      !VerifyDepthwiseBatch(3, 19, 31, 33, 5, 2, 2) ||
      !VerifyDepthwiseBatch(1, 64, 40, 176, 5, 1, 2) ||
      !VerifyDepthwiseBatch(1, 64, 17, 19, 5, 1, 2) ||
      !VerifyDepthwiseScalar(64, 40, 176, 5, 2) ||
      !VerifyDepthwiseScalar(7, 17, 19, 5, 2) ||
      !VerifyFoldedDepthwiseHardSwish(16, 6, 76, 1, 5, 0, 2) ||
      !VerifyFoldedDepthwiseHardSwish(7, 9, 11, 3, 3, 1, 1)) return 1;
  for (int activation = 0; activation < 4; ++activation) {
    if (!VerifyDepthwisePointwiseFused(16, 16, 40, 176, activation) ||
        !VerifyDepthwisePointwiseFused(8, 6, 17, 19, activation) ||
        !VerifyDepthwisePointwiseFused(5, 7, 9, 11, activation)) {
      return 1;
    }
  }
  for (int activation = 0; activation < 4; ++activation) {
    if (!VerifyDepthwisePointwiseFused5x5(64, 16, 40, 176, activation) ||
        !VerifyDepthwisePointwiseFused5x5(7, 5, 9, 11, activation) ||
        !VerifyDepthwisePointwiseFused5x5(16, 8, 17, 19, activation)) {
      return 1;
    }
  }
  if (!VerifyPointwiseDepthwise(16, 32, 40, 176, true) ||
      !VerifyPointwiseDepthwise(16, 32, 40, 176, false) ||
      !VerifyPointwiseDepthwise(16, 32, 33, 17, true) ||
      !VerifyPointwiseDepthwise(8, 8, 32, 16, false) ||
      !VerifyPointwiseDepthwise(7, 9, 9, 11, true)) {
    return 1;
  }
  if (!VerifyDepthwiseExpandGelu(16, 32, 40, 176) ||
      !VerifyDepthwiseExpandGelu(16, 32, 33, 65) ||
      !VerifyDepthwiseExpandGelu(8, 16, 17, 19) ||
      !VerifyDepthwiseExpandGelu(5, 7, 9, 11) ||
      !VerifyDepthwiseExpandGelu(48, 96, 12, 76) ||
      !VerifyDepthwiseExpandGelu(160, 320, 6, 76)) {
    return 1;
  }
  if (!VerifyConvBatch(3, 3, 16, 41, 39, 3, 3, 2, 2, 1, 1, true) ||
      !VerifyConvBatch(3, 7, 9, 37, 35, 3, 3, 1, 1, 1, 1, false) ||
      !VerifyConvBatch(3, 32, 8, 128, 127, 3, 3, 1, 1, 1, 1, true) ||
      !VerifyConvBatch(3, 11, 13, 39, 41, 3, 3, 2, 2, 1, 1, true) ||
      !VerifyConvBatch(3, 11, 9, 39, 41, 2, 2, 1, 1, 0, 0, true) ||
      !VerifyConvBatch(3, 32, 8, 65, 63, 5, 5, 1, 1, 2, 2, false) ||
      !VerifyConvBatch(3, 32, 9, 65, 63, 7, 7, 1, 1, 3, 3, false) ||
      !VerifyConvBatch(3, 13, 9, 39, 41, 1, 5, 1, 1, 0, 2, false) ||
      !VerifyConvBatch(3, 13, 9, 39, 41, 5, 1, 1, 1, 2, 0, false)) return 1;
  if (!VerifyPointwiseHardSwishBatch(3, 7, 9, 37) ||
      !VerifyPointwiseHardSwishBatch(3, 64, 64, 1024)) return 1;
  if (!VerifySqueezeExcitation(1, 16, 4, 40, 176, false) ||
      !VerifySqueezeExcitation(1, 16, 4, 40, 176, true) ||
      !VerifySqueezeExcitation(1, 8, 2, 17, 19, true) ||
      !VerifySqueezeExcitation(2, 24, 6, 12, 37, false)) return 1;
  if (!VerifyConvTransposeBatch(3, 7, 9, 17, 19) ||
      !VerifyConvTransposeBatch(3, 32, 32, 64, 63) ||
      !VerifyConvTransposeScalar(16, 16, 40, 176) ||
      !VerifyConvTransposeScalar(16, 16, 80, 352) ||
      !VerifyConvTransposeAct(16, 16, 40, 176, 1) ||
      !VerifyConvTransposeAct(16, 16, 80, 352, 2) ||
      !VerifyConvTransposeAct(7, 9, 17, 19, 1) ||
      !VerifyConvTransposeChain(16, 16, 1, 9, 11) ||
      !VerifyConvTransposeChain(16, 16, 1, 40, 176) ||
      !VerifyConvTransposeChain(7, 9, 2, 5, 6)) return 1;
  if (!VerifyBatchNormAffine(3, 7, 37) || !VerifyBatchNormAffine(3, 64, 1024)) return 1;
  if (!VerifyNearestResize2xAdd(1, 3, 17, 19) ||
      !VerifyNearestResize2xAdd(2, 64, 64, 63) ||
      !VerifyNearestResize2xAdd(1, 16, 20, 88)) return 1;
  if (!VerifyAveragePool3x2Valid(3, 17, 19) ||
      !VerifyAveragePool3x2Valid(64, 48, 127)) return 1;
  if (!VerifyCtcTop1(3, 17, 31) || !VerifyCtcTop1(4, 41, 6906) ||
      !VerifyCtcTop1FirstMaximum()) return 1;
  if (!VerifyGemmCtcTop1(1, 17, 31, 29) || !VerifyGemmCtcTop1(4, 41, 6906, 192) ||
      !VerifyGemmCtcTop1(1, 4, 18710, 192) || !VerifyGemmCtcTop1(1, 16, 6906, 80)) return 1;
  if (std::getenv("PPOCR_APPROX_GELU") != nullptr && !VerifyApproximateGelu()) return 1;
  std::cout << "kernel RGB-upload/identity-rgb/LayerNorm/GEMM/2x2/3x3/residual-pointwise/depthwise/conv/hardswish/transpose/resize-add/BN/GELU/exact-GELU/CTC batch tile/tail smoke passed\n";
}