From 3bacafbf7d8a73e4edc55eea6f0e49ce0fd6bdd2 Mon Sep 17 00:00:00 2001 From: 076923 <076923@naver.com> Date: Mon, 9 Nov 2026 01:08:52 +0900 Subject: [PATCH 1/2] =?UTF-8?q?[=EA=B8=B0=EB=8A=A5]=20:=20Lambda=20?= =?UTF-8?q?=EC=97=90=EB=AE=AC=EB=A0=88=EC=9D=B4=ED=84=B0=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=EC=99=80=20=EA=B3=B5=EA=B8=89=EB=A7=9D=20?= =?UTF-8?q?=EA=B2=80=EC=A6=9D=EC=9D=84=20=EC=B6=94=EA=B0=80=ED=95=9C?= =?UTF-8?q?=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 레이어 zip을 읽기 전용 /opt에 풀어 RIE로 handler를 호출하고 모듈 포함·제외, 성능, 메모리를 측정한다 - OpenCV와 NumPy 소스 SHA256을 검증하고 빌드 도구 버전을 고정한다 - 빌드 이미지 digest와 도구 버전을 기록하고 zip마다 CycloneDX SBOM을 만든다 - GitHub Actions를 커밋 SHA로 고정하고 Dependabot과 빌드 provenance attestation을 추가한다 - README에 BLAS, AVX2 주의 사항과 테스트, 공급망 섹션을 추가한다 --- .github/dependabot.yml | 9 ++ .github/workflows/build.yml | 52 +++++++--- README.md | 63 +++++++++++- build.sh | 9 +- scripts/build_layer.py | 196 ++++++++++++++++++++++++++++++++++-- scripts/test_layer.py | 117 +++++++++++++++++++++ tests/lambda/handler.py | 120 ++++++++++++++++++++++ 7 files changed, 534 insertions(+), 32 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 scripts/test_layer.py create mode 100644 tests/lambda/handler.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..478231b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + groups: + actions: + patterns: ["*"] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75bbb05..a1b2f20 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,9 +6,13 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + env: NUMPY_VERSION: "2.5.4" +# Actions are pinned to commit SHAs; Dependabot keeps them up to date. jobs: # cv2 is built with the Python Limited API, so one build per # (OpenCV, profile, arch) serves every Python >= 3.12. @@ -21,12 +25,12 @@ jobs: arch: [x86_64, arm64] runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: ./build.sh 3.12 ${{ matrix.arch }} opencv ${{ matrix.opencv }} ${{ matrix.profile }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: opencv-${{ matrix.opencv }}-${{ matrix.profile }}-${{ matrix.arch }} - path: artifacts/*.tar.gz + path: artifacts/* if-no-files-found: error numpy: @@ -37,15 +41,16 @@ jobs: arch: [x86_64, arm64] runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: ./build.sh ${{ matrix.python }} ${{ matrix.arch }} numpy ${{ env.NUMPY_VERSION }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: numpy-${{ matrix.python }}-${{ matrix.arch }} - path: artifacts/*.tar.gz + path: artifacts/* if-no-files-found: error - # Smoke-test and zip inside each target runtime. + # Smoke-test and zip inside each target runtime, then invoke every zip + # through the Lambda Runtime Interface Emulator with /opt mounted read-only. package: needs: [opencv, numpy] strategy: @@ -55,18 +60,19 @@ jobs: arch: [x86_64, arm64] runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: opencv-*-${{ matrix.arch }} path: artifacts merge-multiple: true - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: numpy-${{ matrix.python }}-${{ matrix.arch }} path: artifacts - run: ./build.sh ${{ matrix.python }} ${{ matrix.arch }} package - - uses: actions/upload-artifact@v4 + - run: python3 scripts/test_layer.py dist + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: layers-${{ matrix.python }}-${{ matrix.arch }} path: dist/* @@ -90,8 +96,10 @@ jobs: runs-on: ubuntu-24.04 permissions: contents: write + id-token: write + attestations: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: layers-* path: dist @@ -101,13 +109,25 @@ jobs: run: | sha256sum *.zip > SHA256SUMS { - echo "| File | Profile | Python | Arch | OpenCV | NumPy | Unzipped (MB) | Zipped (MB) |" - echo "|---|---|---|---|---|---|---|---|" - jq -rs 'sort_by(.file)[] | "| \(.file) | \(.profile) | \(.python) | \(.arch) | \(.opencv) | \(.numpy) | \(.unzipped_mb) | \(.zipped_mb) |"' *.json + echo "| File | Profile | Python | Arch | OpenCV | NumPy | Zipped (MB) | Unzipped (MB) | Import (ms) |" + echo "|---|---|---|---|---|---|---|---|---|" + ls *.json | grep -v '\.cdx\.json$' | xargs jq -rs 'sort_by(.file)[] | "| \(.file) | \(.profile) | \(.python) | \(.arch) | \(.opencv) | \(.numpy) | \(.zipped_mb) | \(.unzipped_mb) | \(.lambda_test.import_ms) |"' + echo + echo "Import time was measured in the Lambda Runtime Interface Emulator on GitHub-hosted runners, not on AWS Lambda." + echo + echo "Each zip has a CycloneDX SBOM (\`.cdx.json\`) and a build provenance attestation:" + echo + echo '```bash' + echo "gh attestation verify .zip -R ${GITHUB_REPOSITORY}" + echo '```' } > ../notes.md - - uses: softprops/action-gh-release@v2 + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: dist/*.zip + - uses: softprops/action-gh-release@8a0cd651e5b2365bd640854a3fa5dfc70e4875f3 # v3.0.4 with: files: | dist/*.zip + dist/*.cdx.json dist/SHA256SUMS body_path: notes.md diff --git a/README.md b/README.md index 448f5a8..bc7b1ed 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ Minimal headless OpenCV layers for AWS Lambda, built from source. - **Small**: OpenCV and its codecs are linked statically into a single stripped `cv2.abi3.so`, without FFmpeg or OpenBLAS. - **Fast cold start**: Precompiled bytecode is included, because Lambda cannot write a bytecode cache to `/opt`. -- **Runtime-matched**: Built and smoke-tested inside the official `public.ecr.aws/lambda/python` images. -- **Reproducible**: Deterministic zips with `SHA256SUMS`, built by GitHub Actions. +- **Runtime-matched**: Built inside the official `public.ecr.aws/lambda/python` images, and every zip is invoked through the Lambda Runtime Interface Emulator before release. +- **Verifiable**: Pinned sources and build tools, deterministic zips, `SHA256SUMS`, a CycloneDX SBOM per zip, and GitHub build provenance attestations. ## Quick Start @@ -105,6 +105,12 @@ The layers keep only what image processing on Lambda needs. | Haar cascade XML files | Ship the XML files you need with your function. `cv2.CascadeClassifier` is only available with 4.14.0, because OpenCV 5 removed it from the main repository | | `python3.11` and older | These runtimes run on Amazon Linux 2 and are not supported | +> [!WARNING] +> **NumPy is built without BLAS.** This is fine for image processing, where NumPy mostly holds pixel arrays for OpenCV. If your function uses NumPy as a compute engine (large matrix products, decompositions, solvers), these layers are the wrong choice. Use the official `opencv-python-headless` and `numpy` wheels, which bundle OpenBLAS. + +> [!WARNING] +> **x86_64 binaries require AVX2.** `cv2` is compiled with `CPU_BASELINE=AVX2` and has no fallback path. Every AWS Lambda x86_64 host supports AVX2, but other machines may not, such as older CPUs or some emulators. On a CPU without AVX2, `import cv2` stops with an error that the build does not support the current CPU. The `arm64` builds use the default NEON baseline. + ## How It Stays Small @@ -118,6 +124,48 @@ The layers keep only what image processing on Lambda needs. | `.pyc` compiled with `unchecked-hash` | Zip rounds file times to 2 seconds, so timestamp-checked bytecode would be ignored on every cold start | +## Testing + +Every zip goes through two checks before it can be merged or released. CI fails if either one fails. + +| Check | Where | What it verifies | +|---|---|---| +| Smoke test | Build container, before zipping | `cv2` and `numpy` import, and basic image operations work | +| Lambda invocation | Fresh `public.ecr.aws/lambda/python` container with the zip extracted to a read-only `/opt`, invoked through the Runtime Interface Emulator with 1024 MB of memory | See below | + +The Lambda invocation runs [`tests/lambda/handler.py`](tests/lambda/handler.py) twice, once cold and once warm, and checks: + +- `cv2` and `numpy` load from `/opt/python`, and the precompiled bytecode is present +- JPEG, PNG, and WebP round trips +- ORB matching recovers a known shift through `findHomography` (`standard`, `dnn`) +- `cv2.dnn.blobFromImage` (`dnn`) +- Each profile has its modules and lacks the ones it excludes, such as `cv2.dnn` in `standard` and `cv2.VideoCapture` everywhere +- Import time, a 12 MP decode → resize → blur → WebP pipeline, and peak memory, which are recorded in each zip's `.json` + +> [!NOTE] +> The emulator runs on GitHub-hosted runners, so the timings are indicative only. Cold start, memory, and latency on AWS Lambda itself are not measured. + +To run the Lambda invocation locally after a build, run `python3 scripts/test_layer.py dist`. + + +## Supply Chain + +| Item | How | +|---|---| +| Source archives | The OpenCV and NumPy source archives are verified against SHA256 hashes pinned in [`scripts/build_layer.py`](scripts/build_layer.py) before building | +| Build tools | CMake, Ninja, Meson, meson-python, and Cython versions are pinned. NumPy builds without build isolation, so only the pinned tools are used | +| Build environment | The Lambda image digest, GCC version, and tool versions are recorded in each zip's `.json` | +| GitHub Actions | Actions are pinned to commit SHAs and updated by Dependabot | +| SBOM | Each release includes a CycloneDX SBOM (`.cdx.json`) per zip, listing OpenCV, NumPy, and the third-party libraries linked into `cv2` | +| Provenance | Each zip has a GitHub build provenance attestation | + +`SHA256SUMS` proves that a download is intact. The attestation also proves that the zip was built by this repository's workflow: + +```bash +gh attestation verify opencv-4.14.0-minimal-python3.13-arm64.zip -R 076923/aws-lambda-python-opencv-layer +``` + + ## Build from Source Requires Docker. Building `arm64` on an `x86_64` machine uses QEMU emulation and is slow. @@ -129,7 +177,12 @@ Requires Docker. Building `arm64` on an `x86_64` machine uses QEMU emulation and ``` dist/opencv-4.14.0-minimal-python3.13-arm64.zip -dist/opencv-4.14.0-minimal-python3.13-arm64.json # versions and sizes +dist/opencv-4.14.0-minimal-python3.13-arm64.json # versions, sizes, build info, test results +dist/opencv-4.14.0-minimal-python3.13-arm64.cdx.json # CycloneDX SBOM +``` + +```bash +python3 scripts/test_layer.py dist # invoke every zip through the Lambda emulator ```
@@ -172,7 +225,9 @@ opencv-{opencv}-{profile}-python{python}-{arch}.zip ## Release -Push a `v*` tag. GitHub Actions builds all 36 zips and attaches them to the release with `SHA256SUMS` and a size table. +Push a `v*` tag. GitHub Actions builds and tests all 36 zips, then attaches them to the release with their SBOMs, `SHA256SUMS`, provenance attestations, and a size table. + +To change the OpenCV or NumPy version, update the workflow matrix and add the SHA256 of the new source archive to `OPENCV_SOURCES` or `NUMPY_SOURCES` in [`scripts/build_layer.py`](scripts/build_layer.py). ```bash git tag v2026.11.0 diff --git a/build.sh b/build.sh index f9f65d7..497c406 100755 --- a/build.sh +++ b/build.sh @@ -22,8 +22,15 @@ esac cd "$(dirname "$0")" mkdir -p artifacts dist +IMAGE="public.ecr.aws/lambda/python:${PY}" +docker pull -q --platform "$PLATFORM" "$IMAGE" > /dev/null +# Record the exact image digest in the build info. +DIGEST=$(docker image inspect --format '{{index .RepoDigests 0}}' "$IMAGE") + docker run --rm --platform "$PLATFORM" \ + -e LAMBDA_IMAGE="$DIGEST" \ + -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ -v "$PWD:/work" -w /work \ --entrypoint python3 \ - "public.ecr.aws/lambda/python:${PY}" \ + "$IMAGE" \ scripts/build_layer.py "$@" diff --git a/scripts/build_layer.py b/scripts/build_layer.py index 6d0a843..9b541ea 100644 --- a/scripts/build_layer.py +++ b/scripts/build_layer.py @@ -11,25 +11,51 @@ -> numpy--python-.tar.gz package Combine every opencv tarball of this arch with this Python's numpy, prune, precompile, smoke-test, and zip. - -> dist/opencv---python-.zip (+ .json) + -> dist/opencv---python-.zip + (+ .json build info, + .cdx.json CycloneDX SBOM) + +Each build stage also writes a .build.json sidecar with the verified +source hash, build tool versions, and bundled third-party libraries. """ import argparse import compileall +import hashlib import json import os import platform +import re import shutil import subprocess import sys import sysconfig import tarfile import urllib.request +import uuid import zipfile from pathlib import Path DEFAULT_NUMPY = "2.5.4" +# Build tools installed with pip inside the build container. The compiler +# comes from the Amazon Linux 2023 repositories of the pinned base image. +BUILD_TOOLS = { + "cmake": "4.4.4", + "ninja": "1.13.2", + "meson": "1.12.1", + "meson-python": "0.22.1", + "cython": "3.3.1", +} + +# SHA256 of the source archives. Add an entry when bumping a version. +OPENCV_SOURCES = { + "4.14.0": "ee8fb9b30eb60850431b4656447080e3737b56e45719c92b67f245950609f86e", + "5.1.0": "638c1dae7e37e45afea21aa217dc667ad7cf57acc66cdf652c074f1e0e5ad6d2", +} +NUMPY_SOURCES = { + "2.5.4": "9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", +} + # OpenCV 5 renamed several modules. Names that do not exist in the source # tree are dropped before configuring, so one list covers 4.x and 5.x. MINIMAL = ["core", "imgproc", "imgcodecs"] @@ -153,7 +179,60 @@ def pip(*args) -> None: def install_toolchain() -> None: run("dnf", "install", "-y", "gcc", "gcc-c++", "binutils") - pip("install", "cmake", "ninja") + pip("install", *[f"{k}=={v}" for k, v in BUILD_TOOLS.items()]) + + +def output(*cmd) -> str: + return subprocess.run([str(c) for c in cmd], check=True, capture_output=True, text=True).stdout + + +def toolchain_info() -> dict: + return { + # Set by build.sh from `docker image inspect`, e.g. ...python@sha256:... + "image": os.environ.get("LAMBDA_IMAGE", "unknown"), + "gcc": output("gcc", "-dumpfullversion").strip(), + "python": platform.python_version(), + **BUILD_TOOLS, + } + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def download_verified(url: str, dest: Path, expected: str) -> None: + if not dest.exists(): + print("+ download", url, flush=True) + urllib.request.urlretrieve(url, dest) + actual = sha256(dest) + if actual != expected: + dest.unlink() + sys.exit(f"SHA256 mismatch for {url}\n expected {expected}\n actual {actual}") + print(f"+ verified sha256 {actual}", flush=True) + + +def sidecar(tarball: Path) -> Path: + return tarball.with_name(tarball.name.removesuffix(".tar.gz") + ".build.json") + + +# Third-party libraries OpenCV builds from its bundled sources, as reported in +# the "Media I/O" and "Other third-party libraries" sections of cmake output. +THIRD_PARTY = re.compile( + r"^(?:--)?\s+(ZLib|JPEG|WEBP|PNG|Protobuf|Flatbuffers|KleidiCV|QUIRC)\s*:\s+(.+?)\s*$", re.M +) + + +def third_party(cmake_output: str) -> dict: + found = {} + for name, value in THIRD_PARTY.findall(cmake_output): + if value.upper() in ("NO", "OFF"): + continue + found[name] = value + return found def strip_binaries(root: Path) -> None: @@ -186,6 +265,8 @@ def unpack(src: Path, dest: Path) -> None: def build_opencv(version: str, profile: str, out: Path) -> Path: + if version not in OPENCV_SOURCES: + sys.exit(f"no SHA256 for OpenCV {version}; add it to OPENCV_SOURCES") install_toolchain() # numpy headers for the bindings; numpy >= 2 headers stay ABI-compatible. pip("install", f"numpy=={DEFAULT_NUMPY}") @@ -193,10 +274,8 @@ def build_opencv(version: str, profile: str, out: Path) -> Path: src_root = Path("/tmp/opencv-src") shutil.rmtree(src_root, ignore_errors=True) tarball = Path(f"/tmp/opencv-{version}.tar.gz") - if not tarball.exists(): - url = f"https://github.com/opencv/opencv/archive/refs/tags/{version}.tar.gz" - print("+ download", url, flush=True) - urllib.request.urlretrieve(url, tarball) + url = f"https://github.com/opencv/opencv/archive/refs/tags/{version}.tar.gz" + download_verified(url, tarball, OPENCV_SOURCES[version]) unpack(tarball, src_root) src = src_root / f"opencv-{version}" @@ -220,8 +299,11 @@ def build_opencv(version: str, profile: str, out: Path) -> Path: "PYTHON3_INCLUDE_DIR": sysconfig.get_paths()["include"], "PYTHON3_NUMPY_INCLUDE_DIRS": numpy.get_include(), } - run("cmake", "-S", src, "-B", build, "-G", "Ninja", - *[f"-D{k}={v}" for k, v in defs.items()]) + configure = [str(c) for c in ("cmake", "-S", src, "-B", build, "-G", "Ninja", + *[f"-D{k}={v}" for k, v in defs.items()])] + print("+", " ".join(configure), flush=True) + result = subprocess.run(configure, check=True, capture_output=True, text=True) + print(result.stdout, result.stderr, sep="\n", flush=True) run("cmake", "--build", build) run("cmake", "--install", build, "--strip") @@ -237,6 +319,12 @@ def build_opencv(version: str, profile: str, out: Path) -> Path: dest = out / f"opencv-{version}-{profile}-{arch()}.tar.gz" pack(stage, dest) + sidecar(dest).write_text(json.dumps({ + "opencv": {"version": version, "url": url, "sha256": OPENCV_SOURCES[version]}, + "modules": modules, + "third_party": third_party(result.stdout), + "toolchain": toolchain_info(), + }, indent=2) + "\n") print(f"{dest.name}: {mb(dir_size(stage))} MB unpacked") return dest @@ -245,20 +333,31 @@ def build_opencv(version: str, profile: str, out: Path) -> Path: def build_numpy(version: str, out: Path) -> Path: + if version not in NUMPY_SOURCES: + sys.exit(f"no SHA256 for numpy {version}; add it to NUMPY_SOURCES") install_toolchain() + sdist = Path(f"/tmp/numpy-{version}.tar.gz") + url = f"https://files.pythonhosted.org/packages/source/n/numpy/numpy-{version}.tar.gz" + download_verified(url, sdist, NUMPY_SOURCES[version]) + stage = Path("/tmp/numpy-stage") shutil.rmtree(stage, ignore_errors=True) # No BLAS/LAPACK: numpy falls back to its bundled lapack_lite, which drops # the ~27 MB OpenBLAS + gfortran runtime. Large matmul/linalg get slower. - pip("install", "--target", stage, "--no-deps", "--no-binary=numpy", + # --no-build-isolation builds with the pinned meson/cython from BUILD_TOOLS. + pip("install", "--target", stage, "--no-deps", "--no-build-isolation", "--config-settings=setup-args=-Dallow-noblas=true", "--config-settings=setup-args=-Dblas=none", "--config-settings=setup-args=-Dlapack=none", - f"numpy=={version}") + sdist) strip_binaries(stage) dest = out / f"numpy-{version}-python{py_version()}-{arch()}.tar.gz" pack(stage, dest) + sidecar(dest).write_text(json.dumps({ + "numpy": {"version": version, "url": url, "sha256": NUMPY_SOURCES[version], "blas": "none"}, + "toolchain": toolchain_info(), + }, indent=2) + "\n") print(f"{dest.name}: {mb(dir_size(stage))} MB unpacked") return dest @@ -323,6 +422,8 @@ def package_one(cv_tar: Path, np_tar: Path, out: Path) -> dict: zip_path = out / f"opencv-{cv_ver}-{profile}-python{py_version()}-{arch()}.zip" make_zip(site, zip_path) + cv_build = json.loads(sidecar(cv_tar).read_text()) if sidecar(cv_tar).exists() else {} + np_build = json.loads(sidecar(np_tar).read_text()) if sidecar(np_tar).exists() else {} info = { "file": zip_path.name, "profile": profile, @@ -331,12 +432,68 @@ def package_one(cv_tar: Path, np_tar: Path, out: Path) -> dict: **versions, "unzipped_mb": mb(dir_size(site)), "zipped_mb": mb(zip_path.stat().st_size), + "sha256": sha256(zip_path), + "build": {"opencv": cv_build, "numpy": np_build}, } zip_path.with_suffix(".json").write_text(json.dumps(info, indent=2) + "\n") - print(json.dumps(info), flush=True) + zip_path.with_suffix(".cdx.json").write_text(json.dumps(make_sbom(info), indent=2) + "\n") + print(json.dumps({k: v for k, v in info.items() if k != "build"}), flush=True) return info +def make_sbom(info: dict) -> dict: + """CycloneDX 1.5 SBOM for one layer zip.""" + cv = info["build"]["opencv"] + np_ = info["build"]["numpy"] + components = [ + { + "type": "library", + "name": "opencv", + "version": info["opencv"], + "purl": f"pkg:github/opencv/opencv@{info['opencv']}", + "licenses": [{"license": {"id": "Apache-2.0"}}], + "hashes": [{"alg": "SHA-256", "content": cv.get("opencv", {}).get("sha256", "")}], + "externalReferences": [{"type": "distribution", "url": cv.get("opencv", {}).get("url", "")}], + "properties": [{"name": "opencv:modules", "value": ",".join(cv.get("modules", []))}], + }, + { + "type": "library", + "name": "numpy", + "version": info["numpy"], + "purl": f"pkg:pypi/numpy@{info['numpy']}", + "licenses": [{"license": {"id": "BSD-3-Clause"}}], + "hashes": [{"alg": "SHA-256", "content": np_.get("numpy", {}).get("sha256", "")}], + "externalReferences": [{"type": "distribution", "url": np_.get("numpy", {}).get("url", "")}], + "properties": [{"name": "numpy:blas", "value": "none"}], + }, + ] + # Bundled by OpenCV and linked statically into cv2.abi3.so. + for name, value in cv.get("third_party", {}).items(): + m = re.search(r"\d+(?:\.\d+)+[^\s,)]*", value) + components.append({ + "type": "library", + "name": name.lower(), + "version": m.group(0) if m else value, + "description": f"Bundled with OpenCV, linked statically: {value}", + }) + toolchain = cv.get("toolchain") or np_.get("toolchain") or {} + return { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "serialNumber": f"urn:uuid:{uuid.uuid5(uuid.NAMESPACE_URL, info['sha256'])}", + "version": 1, + "metadata": { + "component": { + "type": "library", + "name": info["file"], + "hashes": [{"alg": "SHA-256", "content": info["sha256"]}], + }, + "properties": [{"name": f"build:{k}", "value": str(v)} for k, v in toolchain.items()], + }, + "components": components, + } + + def package(artifacts: Path, out: Path) -> None: np_tars = sorted(artifacts.glob(f"numpy-*-python{py_version()}-{arch()}.tar.gz")) cv_tars = sorted(artifacts.glob(f"opencv-*-{arch()}.tar.gz")) @@ -376,6 +533,23 @@ def main() -> None: p.add_argument("--numpy", default=DEFAULT_NUMPY) args = parser.parse_args() + try: + run_stage(args) + finally: + give_back(args.artifacts, args.out) + + +def give_back(*dirs: Path) -> None: + """The container runs as root; hand outputs back to the host user.""" + uid, gid = os.environ.get("HOST_UID"), os.environ.get("HOST_GID") + if os.geteuid() != 0 or not uid or not gid: + return + for d in dirs: + for p in [d, *d.rglob("*")] if d.exists() else []: + os.chown(p, int(uid), int(gid)) + + +def run_stage(args: argparse.Namespace) -> None: if args.stage == "opencv": build_opencv(args.version, args.profile, args.artifacts) elif args.stage == "numpy": diff --git a/scripts/test_layer.py b/scripts/test_layer.py new file mode 100644 index 0000000..bd6a4e5 --- /dev/null +++ b/scripts/test_layer.py @@ -0,0 +1,117 @@ +"""Invoke every layer zip through the Lambda Runtime Interface Emulator. + +For each dist/*.zip: extract it, mount it read-only at /opt in a fresh +public.ecr.aws/lambda/python container, mount tests/lambda at /var/task, and +invoke handler.handler twice (cold, then warm). Results are merged into the +zip's .json under "lambda_test". + +Runs on the host and requires Docker. Usage: python3 scripts/test_layer.py [dist] +""" + +import json +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +PORT = 9000 +URL = f"http://localhost:{PORT}/2015-03-31/functions/function/invocations" +PLATFORMS = {"x86_64": "linux/amd64", "arm64": "linux/arm64"} +MEMORY = "1024m" # emulate a 1024 MB function + + +def invoke(payload: dict, timeout: float) -> tuple[dict, float]: + data = json.dumps(payload).encode() + deadline = time.monotonic() + timeout + while True: + try: + t0 = time.perf_counter() + req = urllib.request.Request(URL, data=data, method="POST") + with urllib.request.urlopen(req, timeout=120) as resp: + body = json.loads(resp.read()) + return body, round((time.perf_counter() - t0) * 1000, 1) + except (urllib.error.URLError, ConnectionError): + if time.monotonic() > deadline: + raise + time.sleep(0.5) + + +def test_zip(zip_path: Path) -> list[str]: + meta_path = zip_path.with_suffix(".json") + meta = json.loads(meta_path.read_text()) + image = f"public.ecr.aws/lambda/python:{meta['python']}" + + with tempfile.TemporaryDirectory() as tmp: + opt = Path(tmp) / "opt" + with zipfile.ZipFile(zip_path) as zf: + zf.extractall(opt) + opt.chmod(0o755) + + container = subprocess.run( + ["docker", "run", "-d", "--rm", + "--platform", PLATFORMS[meta["arch"]], + "--memory", MEMORY, + "-p", f"{PORT}:8080", + "-v", f"{opt}:/opt:ro", + "-v", f"{ROOT / 'tests' / 'lambda'}:/var/task:ro", + image, "handler.handler"], + check=True, capture_output=True, text=True, + ).stdout.strip() + try: + cold, cold_ms = invoke({"profile": meta["profile"]}, timeout=60) + warm, warm_ms = invoke({"profile": meta["profile"]}, timeout=10) + finally: + logs = subprocess.run(["docker", "logs", container], capture_output=True, text=True) + subprocess.run(["docker", "stop", container], capture_output=True) + + if "errorMessage" in cold: + print(logs.stdout, logs.stderr, sep="\n", file=sys.stderr) + return [f"{cold.get('errorType')}: {cold.get('errorMessage')}"] + + errors = list(cold["errors"]) + for key in ("opencv", "numpy", "python"): + expected = meta[key] + if key == "python": + ok = cold[key].startswith(expected + ".") + else: + ok = cold[key] == expected + if not ok: + errors.append(f"{key}: expected {expected}, got {cold[key]}") + + meta["lambda_test"] = { + "memory": MEMORY, + "import_ms": cold["import_ms"], + "cold_invoke_ms": cold_ms, + "warm_invoke_ms": warm_ms, + "pipeline_12mp_ms": warm["pipeline_ms"], + "peak_rss_mb": warm["peak_rss_mb"], + } + meta_path.write_text(json.dumps(meta, indent=2) + "\n") + print(json.dumps({"file": meta["file"], **meta["lambda_test"], "errors": errors}), flush=True) + return errors + + +def main() -> None: + dist = Path(sys.argv[1] if len(sys.argv) > 1 else "dist") + zips = sorted(dist.glob("*.zip")) + if not zips: + sys.exit(f"no zips in {dist}") + if not shutil.which("docker"): + sys.exit("docker is required") + failed = {} + for z in zips: + errors = test_zip(z) + if errors: + failed[z.name] = errors + if failed: + sys.exit("failed:\n" + json.dumps(failed, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/lambda/handler.py b/tests/lambda/handler.py new file mode 100644 index 0000000..7a6a1d3 --- /dev/null +++ b/tests/lambda/handler.py @@ -0,0 +1,120 @@ +"""Lambda handler used by scripts/test_layer.py. + +The layer zip is extracted to /opt (read-only) and this file is mounted at +/var/task, so imports resolve exactly as they do on AWS Lambda. +""" + +import os +import resource +import sys +import time + +_t0 = time.perf_counter() +import cv2 # noqa: E402 +import numpy as np # noqa: E402 + +IMPORT_MS = round((time.perf_counter() - _t0) * 1000, 1) + +# Attributes each profile must provide, and attributes it must not. +PRESENT = { + "minimal": ["resize", "GaussianBlur", "cvtColor", "imencode", "imdecode"], + "standard": ["ORB_create", "findHomography", "QRCodeDetector", "inpaint", "calcOpticalFlowPyrLK"], + "dnn": ["dnn"], +} +ABSENT = { + "minimal": ["ORB_create", "QRCodeDetector", "dnn", "VideoCapture", "VideoWriter"], + "standard": ["dnn", "VideoCapture", "VideoWriter"], + "dnn": ["VideoCapture", "VideoWriter"], +} +INCLUDES = {"minimal": ["minimal"], "standard": ["minimal", "standard"], "dnn": ["minimal", "standard", "dnn"]} + + +def check_layout(errors): + if not cv2.__file__.startswith("/opt/python/"): + errors.append(f"cv2 loaded from {cv2.__file__}") + if not np.__file__.startswith("/opt/python/"): + errors.append(f"numpy loaded from {np.__file__}") + if os.access("/opt/python", os.W_OK): + errors.append("/opt/python is writable; the test must mount it read-only") + # Precompiled bytecode must ship with the layer, since /opt is read-only. + if not (np.__cached__ and os.path.exists(np.__cached__)): + errors.append(f"missing bytecode {np.__cached__}") + + +def check_modules(profile, errors): + for p in INCLUDES[profile]: + for name in PRESENT[p]: + if not hasattr(cv2, name): + errors.append(f"cv2.{name} missing") + for name in ABSENT[profile]: + if hasattr(cv2, name): + errors.append(f"cv2.{name} should not be in the {profile} profile") + + +def check_codecs(errors): + img = np.zeros((64, 64, 3), np.uint8) + cv2.circle(img, (32, 32), 10, (255, 255, 255), -1) + for ext in (".jpg", ".png", ".webp"): + ok, buf = cv2.imencode(ext, img) + if not ok or cv2.imdecode(buf, cv2.IMREAD_COLOR).shape != img.shape: + errors.append(f"{ext} round trip failed") + + +def check_features(profile, errors): + if profile == "minimal": + return + rng = np.random.default_rng(0) + gray = (rng.random((480, 640)) * 255).astype(np.uint8) + gray = cv2.GaussianBlur(gray, (5, 5), 0) + shifted = np.roll(gray, (7, 11), axis=(0, 1)) + orb = cv2.ORB_create(500) + k1, d1 = orb.detectAndCompute(gray, None) + k2, d2 = orb.detectAndCompute(shifted, None) + matches = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True).match(d1, d2) + if len(matches) < 4: + errors.append(f"too few ORB matches: {len(matches)}") + return + src = np.float32([k1[m.queryIdx].pt for m in matches]) + dst = np.float32([k2[m.trainIdx].pt for m in matches]) + h, _ = cv2.findHomography(src, dst, cv2.RANSAC, 3.0) + if h is None or abs(h[0, 2] - 11) > 1 or abs(h[1, 2] - 7) > 1: + errors.append(f"unexpected homography {h}") + if profile == "dnn": + blob = cv2.dnn.blobFromImage(cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR), 1 / 255, (224, 224)) + if blob.shape != (1, 3, 224, 224): + errors.append(f"unexpected blob shape {blob.shape}") + + +def large_image(): + """Typical photo workload: 12 MP decode, resize, blur, encode.""" + rng = np.random.default_rng(0) + photo = (rng.random((3024, 4032, 3)) * 255).astype(np.uint8) + ok, jpeg = cv2.imencode(".jpg", photo, [cv2.IMWRITE_JPEG_QUALITY, 90]) + t0 = time.perf_counter() + img = cv2.imdecode(jpeg, cv2.IMREAD_COLOR) + img = cv2.resize(img, (1024, 768), interpolation=cv2.INTER_AREA) + img = cv2.GaussianBlur(img, (5, 5), 0) + ok, out = cv2.imencode(".webp", img, [cv2.IMWRITE_WEBP_QUALITY, 80]) + return round((time.perf_counter() - t0) * 1000, 1) + + +def handler(event, context): + profile = event["profile"] + errors = [] + check_layout(errors) + check_modules(profile, errors) + check_codecs(errors) + check_features(profile, errors) + if np.linalg.inv(np.eye(3) * 2).trace() != 1.5: + errors.append("numpy linalg failed") + pipeline_ms = large_image() + return { + "ok": not errors, + "errors": errors, + "opencv": cv2.__version__, + "numpy": np.__version__, + "python": sys.version.split()[0], + "import_ms": IMPORT_MS, + "pipeline_ms": pipeline_ms, + "peak_rss_mb": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1), + } From 423932537fe7c313bbddf71a8c8d8399d846a1ea Mon Sep 17 00:00:00 2001 From: 076923 <076923@naver.com> Date: Mon, 9 Nov 2026 01:23:48 +0900 Subject: [PATCH 2/2] =?UTF-8?q?[=EA=B0=9C=EC=84=A0]=20:=20x86=5F64=20libjp?= =?UTF-8?q?eg-turbo=20SIMD=EB=A5=BC=20=ED=99=9C=EC=84=B1=ED=99=94=ED=95=9C?= =?UTF-8?q?=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NASM이 없어 x86_64에서 JPEG SIMD가 꺼진 채 빌드되던 문제를 고치고, SIMD가 꺼지면 빌드를 실패시킨다 --- scripts/build_layer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/build_layer.py b/scripts/build_layer.py index 9b541ea..6c1bd2f 100644 --- a/scripts/build_layer.py +++ b/scripts/build_layer.py @@ -178,7 +178,10 @@ def pip(*args) -> None: def install_toolchain() -> None: - run("dnf", "install", "-y", "gcc", "gcc-c++", "binutils") + # libjpeg-turbo needs NASM for its x86_64 SIMD code; without it JPEG + # encode/decode silently falls back to plain C. arm64 uses NEON intrinsics. + extra = ["nasm"] if arch() == "x86_64" else [] + run("dnf", "install", "-y", "gcc", "gcc-c++", "binutils", *extra) pip("install", *[f"{k}=={v}" for k, v in BUILD_TOOLS.items()]) @@ -304,6 +307,9 @@ def build_opencv(version: str, profile: str, out: Path) -> Path: print("+", " ".join(configure), flush=True) result = subprocess.run(configure, check=True, capture_output=True, text=True) print(result.stdout, result.stderr, sep="\n", flush=True) + jpeg_simd = re.search(r"JPEG:.*\n.*SIMD Support Request:.*\n--\s+SIMD Support:\s+(\S+)", result.stdout) + if not jpeg_simd or jpeg_simd.group(1) != "YES": + sys.exit("libjpeg-turbo SIMD is disabled; check the cmake output above") run("cmake", "--build", build) run("cmake", "--install", build, "--strip")