diff --git a/.gitattributes b/.gitattributes index 486a232..6313b56 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -*.zip filter=lfs diff=lfs merge=lfs -text +* text=auto eol=lf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..75bbb05 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,113 @@ +name: Build layers + +on: + push: + tags: ["v*"] + pull_request: + workflow_dispatch: + +env: + NUMPY_VERSION: "2.5.4" + +jobs: + # cv2 is built with the Python Limited API, so one build per + # (OpenCV, profile, arch) serves every Python >= 3.12. + opencv: + strategy: + fail-fast: false + matrix: + opencv: ["4.14.0", "5.1.0"] + profile: [minimal, standard, dnn] + arch: [x86_64, arm64] + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + steps: + - uses: actions/checkout@v4 + - run: ./build.sh 3.12 ${{ matrix.arch }} opencv ${{ matrix.opencv }} ${{ matrix.profile }} + - uses: actions/upload-artifact@v4 + with: + name: opencv-${{ matrix.opencv }}-${{ matrix.profile }}-${{ matrix.arch }} + path: artifacts/*.tar.gz + if-no-files-found: error + + numpy: + strategy: + fail-fast: false + matrix: + python: ["3.12", "3.13", "3.14"] + arch: [x86_64, arm64] + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + steps: + - uses: actions/checkout@v4 + - run: ./build.sh ${{ matrix.python }} ${{ matrix.arch }} numpy ${{ env.NUMPY_VERSION }} + - uses: actions/upload-artifact@v4 + with: + name: numpy-${{ matrix.python }}-${{ matrix.arch }} + path: artifacts/*.tar.gz + if-no-files-found: error + + # Smoke-test and zip inside each target runtime. + package: + needs: [opencv, numpy] + strategy: + fail-fast: false + matrix: + python: ["3.12", "3.13", "3.14"] + arch: [x86_64, arm64] + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: opencv-*-${{ matrix.arch }} + path: artifacts + merge-multiple: true + - uses: actions/download-artifact@v4 + with: + name: numpy-${{ matrix.python }}-${{ matrix.arch }} + path: artifacts + - run: ./build.sh ${{ matrix.python }} ${{ matrix.arch }} package + - uses: actions/upload-artifact@v4 + with: + name: layers-${{ matrix.python }}-${{ matrix.arch }} + path: dist/* + if-no-files-found: error + + # Single required status check for branch protection. Fails if any job + # failed, was cancelled, or was skipped because an earlier stage failed. + result: + name: CI result + if: always() + needs: [opencv, numpy, package] + runs-on: ubuntu-24.04 + steps: + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') + run: exit 1 + - run: echo "all jobs passed" + + release: + if: startsWith(github.ref, 'refs/tags/v') + needs: package + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: layers-* + path: dist + merge-multiple: true + - name: Checksums and release notes + working-directory: dist + run: | + sha256sum *.zip > SHA256SUMS + { + echo "| File | Profile | Python | Arch | OpenCV | NumPy | Unzipped (MB) | Zipped (MB) |" + echo "|---|---|---|---|---|---|---|---|" + jq -rs 'sort_by(.file)[] | "| \(.file) | \(.profile) | \(.python) | \(.arch) | \(.opencv) | \(.numpy) | \(.unzipped_mb) | \(.zipped_mb) |"' *.json + } > ../notes.md + - uses: softprops/action-gh-release@v2 + with: + files: | + dist/*.zip + dist/SHA256SUMS + body_path: notes.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..125e966 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +artifacts/ +dist/ +__pycache__/ diff --git a/LICENSE b/LICENSE index 43b1417..77c9eb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013-2020 Michael Rose and contributors +Copyright (c) 2022-2026 Daehee Yun Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Python 3.7.10/opencv-python-headless-3.4.17.61.zip b/Python 3.7.10/opencv-python-headless-3.4.17.61.zip deleted file mode 100644 index 2fd0836..0000000 --- a/Python 3.7.10/opencv-python-headless-3.4.17.61.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0939cd90ef54c386f3e816e51ec373fb0ba7b4f70acc6df2036b3d3cefc645b3 -size 63750529 diff --git a/Python 3.7.10/opencv-python-headless-4.0.1.24.zip b/Python 3.7.10/opencv-python-headless-4.0.1.24.zip deleted file mode 100644 index c032eab..0000000 --- a/Python 3.7.10/opencv-python-headless-4.0.1.24.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:58b78decc14e1b366698495aaf3cf00a963f2b0eb1896fa14572dc4f37d3e398 -size 37552559 diff --git a/Python 3.7.10/opencv-python-headless-4.1.2.30.zip b/Python 3.7.10/opencv-python-headless-4.1.2.30.zip deleted file mode 100644 index dff7041..0000000 --- a/Python 3.7.10/opencv-python-headless-4.1.2.30.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bff3de06e5e1591ac6b2aa2d6b93bbd15634b31b25fe3f5c8db0cf6c589654b7 -size 40166645 diff --git a/Python 3.7.10/opencv-python-headless-4.2.0.34.zip b/Python 3.7.10/opencv-python-headless-4.2.0.34.zip deleted file mode 100644 index 3707861..0000000 --- a/Python 3.7.10/opencv-python-headless-4.2.0.34.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a708d583902d17e9f1c520329f945e0449be9de95ae2889706d5709cf28d0cc3 -size 40039875 diff --git a/Python 3.7.10/opencv-python-headless-4.3.0.38.zip b/Python 3.7.10/opencv-python-headless-4.3.0.38.zip deleted file mode 100644 index 80ac31c..0000000 --- a/Python 3.7.10/opencv-python-headless-4.3.0.38.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e3abeeae1ecfc22b46a4aa4ae6ad7f6dbe3a6281e3da9e95afba82940972be16 -size 54848113 diff --git a/Python 3.7.10/opencv-python-headless-4.4.0.46.zip b/Python 3.7.10/opencv-python-headless-4.4.0.46.zip deleted file mode 100644 index 4b26678..0000000 --- a/Python 3.7.10/opencv-python-headless-4.4.0.46.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:174c98df8a9aa9a931bee1b76098f94f7ea374e78d9043126ba6dc3bb0e984ef -size 55082921 diff --git a/Python 3.7.10/opencv-python-headless-4.5.5.62.zip b/Python 3.7.10/opencv-python-headless-4.5.5.62.zip deleted file mode 100644 index 8631109..0000000 --- a/Python 3.7.10/opencv-python-headless-4.5.5.62.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:623c7b547533f224c4aea3f205df006197ba77b285428e484ae8f2440aa55763 -size 66124771 diff --git a/README.md b/README.md index a766004..448f5a8 100644 --- a/README.md +++ b/README.md @@ -1,85 +1,187 @@ -## Introduction +
-This repository provides `pre-built headless OpenCV` for AWS Lambda. Python OpenCV includes native code libraries, I compiled and build these libraries using a Linux development machine so that the binaries are compatible with Amazon Linux. Once this layer is applied, you can use the layer to host your own algorithms on lambda for Python OpenCV. +# OpenCV Layer for AWS Lambda +Minimal headless OpenCV layers for AWS Lambda, built from source. -## Python Version +[![Build](https://github.com/076923/aws-lambda-python-opencv-layer/actions/workflows/build.yml/badge.svg)](https://github.com/076923/aws-lambda-python-opencv-layer/actions/workflows/build.yml) +[![Release](https://img.shields.io/github/v/release/076923/aws-lambda-python-opencv-layer)](https://github.com/076923/aws-lambda-python-opencv-layer/releases/latest) +[![Downloads](https://img.shields.io/github/downloads/076923/aws-lambda-python-opencv-layer/total)](https://github.com/076923/aws-lambda-python-opencv-layer/releases) +![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13%20%7C%203.14-blue) +![Arch](https://img.shields.io/badge/arch-x86__64%20%7C%20arm64-orange) +[![License](https://img.shields.io/github/license/076923/aws-lambda-python-opencv-layer)](LICENSE) -For each Python directory, it was created as follows: +
-### Support Pre-built Python 3.7.10 +- **Small**: OpenCV and its codecs are linked statically into a single stripped `cv2.abi3.so`, without FFmpeg or OpenBLAS. +- **Fast cold start**: Precompiled bytecode is included, because Lambda cannot write a bytecode cache to `/opt`. +- **Runtime-matched**: Built and smoke-tested inside the official `public.ecr.aws/lambda/python` images. +- **Reproducible**: Deterministic zips with `SHA256SUMS`, built by GitHub Actions. -> `based on Numpy==1.21.5` -- opencv-python-headless-3.4.17.61 -- opencv-python-headless-4.0.1.24 -- opencv-python-headless-4.1.2.30 -- opencv-python-headless-4.2.0.34 -- opencv-python-headless-4.3.0.38 -- opencv-python-headless-4.4.0.46 -- opencv-python-headless-4.5.5.62 +## Quick Start -### Support Pre-built Python 3.8.16 +**1. Download** a zip from the [latest release][latest]. The asset URL is stable across releases. -> `based on Numpy==1.24.2` +```bash +curl -LO https://github.com/076923/aws-lambda-python-opencv-layer/releases/latest/download/opencv-4.14.0-minimal-python3.13-arm64.zip +``` + +**2. Publish** it as a layer. The runtime and architecture must match your function. + +```bash +aws lambda publish-layer-version \ + --layer-name opencv-minimal \ + --zip-file fileb://opencv-4.14.0-minimal-python3.13-arm64.zip \ + --compatible-runtimes python3.13 \ + --compatible-architectures arm64 +``` + +> [!NOTE] +> Every zip is under the 50 MB limit for direct upload. If your own build is larger, upload it to S3 first and pass `--content S3Bucket=,S3Key=`. + +**3. Use** it in your function. + +```python +import base64 + +import cv2 +import numpy as np + + +def handler(event, context): + data = np.frombuffer(base64.b64decode(event["image"]), np.uint8) + img = cv2.imdecode(data, cv2.IMREAD_COLOR) + thumb = cv2.resize(img, (256, 256), interpolation=cv2.INTER_AREA) + ok, buf = cv2.imencode(".webp", thumb, [cv2.IMWRITE_WEBP_QUALITY, 80]) + return {"image": base64.b64encode(buf).decode()} +``` + + +## Choose a Variant + +Asset name: `opencv-{opencv}-{profile}-python{python}-{arch}.zip` + +| Option | Values | +|---|---| +| `opencv` | `4.14.0`, `5.1.0` | +| `profile` | `minimal`, `standard`, `dnn` | +| `python` | `3.12`, `3.13`, `3.14` | +| `arch` | `x86_64`, `arm64` | + +| Profile | Modules | Use for | +|---|---|---| +| `minimal` | core, imgproc, imgcodecs | Resize, crop, color conversion, filters, encode and decode | +| `standard` | minimal + features2d, calib3d, flann, objdetect, photo, video, ml | Feature matching, homography, QR codes, inpainting, optical flow | +| `dnn` | standard + dnn | Running ONNX and other models with `cv2.dnn` | + +Sizes for Python 3.13 with NumPy 2.5.4 (zipped / unzipped, MB). Other Python versions differ by less than 0.3 MB. -- opencv-python-headless-3.4.18.65 -- opencv-python-headless-4.1.2.30 -- opencv-python-headless-4.2.0.34 -- opencv-python-headless-4.3.0.38 -- opencv-python-headless-4.4.0.46 -- opencv-python-headless-4.5.5.64 -- opencv-python-headless-4.6.0.66 -- opencv-python-headless-4.7.0.68 +| Profile | 4.14.0 x86_64 | 4.14.0 arm64 | 5.1.0 x86_64 | 5.1.0 arm64 | +|---|---|---|---|---| +| `minimal` | 12.4 / 33.6 | 10.5 / 29.2 | 14.6 / 39.3 | 13.0 / 35.1 | +| `standard` | 15.7 / 41.7 | 13.4 / 36.0 | 17.0 / 44.9 | 15.0 / 39.8 | +| `dnn` | 18.3 / 48.6 | 15.7 / 42.0 | 20.9 / 57.0 | 18.4 / 50.1 | -### Support Pre-built Python 3.9.16 +For comparison, the official `opencv-python-headless` 4.14.0.94 wheel with NumPy is about 72 MB zipped and 215 MB unzipped. Each release lists the size of every asset. -> `based on Numpy==1.24.2` +> [!TIP] +> `arm64` (Graviton) is about 20% cheaper per GB-second than `x86_64`. -- opencv-python-headless-3.4.18.65 -- opencv-python-headless-4.4.0.46 -- opencv-python-headless-4.5.5.64 -- opencv-python-headless-4.6.0.66 -- opencv-python-headless-4.7.0.68 +> [!NOTE] +> OpenCV 5 renamed some modules (`features2d` → `features`, `calib3d` → `calib`, `geometry`, `stereo`). The `standard` profile includes the equivalent modules for each version. -## Apply Layer +## Limitations -1. Upload the `opencv-python-headless.zip` file to Amazon S3. +The layers keep only what image processing on Lambda needs. -2. Open the Layers page of the Lambda console. +| Not included | Impact | +|---|---| +| FFmpeg, GStreamer, `videoio`, `highgui` | `cv2.VideoCapture`, `cv2.VideoWriter`, and `cv2.imshow` are not available | +| TIFF, JPEG 2000, OpenEXR, AVIF, HDR, PNM codecs | Only JPEG, PNG, and WebP can be read and written | +| OpenBLAS (OpenCV and NumPy) | NumPy uses its bundled `lapack_lite`. Large `matmul` and `linalg` calls are slower | +| IPP, OpenCL, TBB, Eigen | OpenCV uses its own SIMD kernels and pthreads | +| Haar cascade XML files | Ship the XML files you need with your function. `cv2.CascadeClassifier` is only available with 4.14.0, because OpenCV 5 removed it from the main repository | +| `python3.11` and older | These runtimes run on Amazon Linux 2 and are not supported | -3. Choose Create layer. -4. Under Layer configuration, for Name, enter a name for your layer. +## How It Stays Small -5. To upload OpenCV Python Headless layer +| Technique | Effect | +|---|---| +| Static build into one `cv2.abi3.so` | No bundled `.libs` directories or duplicate runtimes | +| `CPU_BASELINE=AVX2` on x86_64 without runtime dispatch | No SSE4, AVX, or AVX-512 copies of the kernels. Every Lambda x86_64 host supports AVX2 | +| `-ffunction-sections`, `--gc-sections`, `strip` | Removes unused code and symbols | +| NumPy without BLAS | Removes about 27 MB of OpenBLAS and gfortran runtime | +| Remove NumPy tests, headers, static libs, `f2py`, `.pyi` stubs | Removes files that are only needed for development | +| `.pyc` compiled with `unchecked-hash` | Zip rounds file times to 2 seconds, so timestamp-checked bytecode would be ignored on every cold start | -6. Choose Upload a file from Amazon S3. Then, for Amazon S3 link URL, enter a link to the file. -7. Choose Create. +## Build from Source +Requires Docker. Building `arm64` on an `x86_64` machine uses QEMU emulation and is slow. -## Structure +```bash +# ./build.sh all [--numpy ] +./build.sh 3.13 arm64 all 4.14.0 minimal +``` -Example file structure for the `opencv-python-headless` library: ``` -opencv-python-headless.{major}.{minor}.{revision}.{package_version}.zip - └ python/bin - └ python/cv2 - └ python/numpy - └ python/numpy.libs - └ python/numpy-1.21.5.dist-info - └ python/opencv_python_headless-{major}.{minor}.{revision}.{package_version}.dist-info - └ python/opencv_python_headless.libs +dist/opencv-4.14.0-minimal-python3.13-arm64.zip +dist/opencv-4.14.0-minimal-python3.13-arm64.json # versions and sizes ``` -## Another layer +
+Build stages + +The build has three stages, which CI runs as separate jobs. + +| Stage | Runs once per | Output | +|---|---|---| +| `opencv` | OpenCV × profile × arch | `cv2` built with the Python Limited API, so one binary serves Python 3.12 and later | +| `numpy` | Python × arch | NumPy without BLAS | +| `package` | Python × arch | Every zip for that runtime, after a smoke test | + +```bash +./build.sh 3.12 x86_64 opencv 4.14.0 standard # artifacts/opencv-4.14.0-standard-x86_64.tar.gz +./build.sh 3.13 x86_64 numpy # artifacts/numpy-2.5.4-python3.13-x86_64.tar.gz +./build.sh 3.13 x86_64 package # dist/*.zip for every built cv2 +``` + +
+ +
+Layer structure + +Lambda extracts the layer to `/opt`, and `/opt/python` is on `sys.path`. + +``` +opencv-{opencv}-{profile}-python{python}-{arch}.zip +└── python/ + ├── cv2/ + │ ├── __init__.py + │ ├── python-3/cv2.abi3.so + │ └── licenses/ + ├── numpy/ + └── numpy-{version}.dist-info/ +``` + +
+ + +## Release + +Push a `v*` tag. GitHub Actions builds all 36 zips and attaches them to the release with `SHA256SUMS` and a size table. + +```bash +git tag v2026.11.0 +git push origin v2026.11.0 +``` -The following [link][lambda-link] show how you can structure the folders in your layer .zip archive. -[lambda-link]: https://docs.aws.amazon.com/en_es/lambda/latest/dg/configuration-layers.html +## Author -## Authors +Daehee Yun ([s076923@gmail.com](mailto:s076923@gmail.com)) -Daehee Yun(s076923@gmail.com) \ No newline at end of file +[latest]: https://github.com/076923/aws-lambda-python-opencv-layer/releases/latest diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..f9f65d7 --- /dev/null +++ b/build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Run a build stage inside the official Lambda image. +# +# Usage: ./build.sh [args...] +# +# ./build.sh 3.13 arm64 all 4.14.0 minimal # one layer, end to end +# ./build.sh 3.12 x86_64 opencv 4.14.0 standard # cv2 only (abi3, any Python >= 3.12) +# ./build.sh 3.13 x86_64 numpy # numpy only +# ./build.sh 3.13 x86_64 package # zip every built cv2 with this numpy +set -euo pipefail + +PY=${1:?python version (3.12, 3.13, 3.14)} +ARCH=${2:?arch (x86_64, arm64)} +shift 2 + +case "$ARCH" in + x86_64) PLATFORM=linux/amd64 ;; + arm64) PLATFORM=linux/arm64 ;; + *) echo "unknown arch: $ARCH" >&2; exit 1 ;; +esac + +cd "$(dirname "$0")" +mkdir -p artifacts dist + +docker run --rm --platform "$PLATFORM" \ + -v "$PWD:/work" -w /work \ + --entrypoint python3 \ + "public.ecr.aws/lambda/python:${PY}" \ + scripts/build_layer.py "$@" diff --git a/python 3.8.16/opencv-python-headless-3.4.18.65.zip b/python 3.8.16/opencv-python-headless-3.4.18.65.zip deleted file mode 100644 index 678d234..0000000 --- a/python 3.8.16/opencv-python-headless-3.4.18.65.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c8af8b6810700b45cedba1cfe12db8201d3e4fca9d46a937d54740f8828b402 -size 66012521 diff --git a/python 3.8.16/opencv-python-headless-4.1.2.30.zip b/python 3.8.16/opencv-python-headless-4.1.2.30.zip deleted file mode 100644 index 0ed525f..0000000 --- a/python 3.8.16/opencv-python-headless-4.1.2.30.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e7adee39f55e45bb4c5df0f3d05c9e7be3f3c06ba08505958b45f4aef48ee467 -size 42021766 diff --git a/python 3.8.16/opencv-python-headless-4.2.0.34.zip b/python 3.8.16/opencv-python-headless-4.2.0.34.zip deleted file mode 100644 index 102878a..0000000 --- a/python 3.8.16/opencv-python-headless-4.2.0.34.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:368a8777fb7cc951707e6230d51703fd25e40d58d3ad9dbef4dd225e825addfc -size 41896828 diff --git a/python 3.8.16/opencv-python-headless-4.3.0.38.zip b/python 3.8.16/opencv-python-headless-4.3.0.38.zip deleted file mode 100644 index c76d7ec..0000000 --- a/python 3.8.16/opencv-python-headless-4.3.0.38.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:47d64b11253e19ed8e7bf4b8cd236736aab6427aafec7785009c658dc7a71ca9 -size 56704935 diff --git a/python 3.8.16/opencv-python-headless-4.4.0.46.zip b/python 3.8.16/opencv-python-headless-4.4.0.46.zip deleted file mode 100644 index da42478..0000000 --- a/python 3.8.16/opencv-python-headless-4.4.0.46.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dfcd5609df4562c6bca8675c4a47afaa3200260a13ae1fbdb6b9cdccba3cc94a -size 56947601 diff --git a/python 3.8.16/opencv-python-headless-4.5.5.64.zip b/python 3.8.16/opencv-python-headless-4.5.5.64.zip deleted file mode 100644 index ea145cd..0000000 --- a/python 3.8.16/opencv-python-headless-4.5.5.64.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0aad537332b332fe2e1e7808caa1f13534255ac5468cf020779d96abe74d1e56 -size 68032208 diff --git a/python 3.8.16/opencv-python-headless-4.6.0.66.zip b/python 3.8.16/opencv-python-headless-4.6.0.66.zip deleted file mode 100644 index 4dc9719..0000000 --- a/python 3.8.16/opencv-python-headless-4.6.0.66.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:75ad24ffa1ab2ee9b468775b7f250c74c3f8fd2b7a6ff59b56ca10f8986b530f -size 68544232 diff --git a/python 3.8.16/opencv-python-headless-4.7.0.68.zip b/python 3.8.16/opencv-python-headless-4.7.0.68.zip deleted file mode 100644 index 99d66e8..0000000 --- a/python 3.8.16/opencv-python-headless-4.7.0.68.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:36b708dded04544ed21da094bf71abe3c7a2e7ba0658a748f9279e7f6ec41829 -size 69504135 diff --git a/python 3.9.16/opencv-python-headless-3.4.18.65.zip b/python 3.9.16/opencv-python-headless-3.4.18.65.zip deleted file mode 100644 index 0e66223..0000000 --- a/python 3.9.16/opencv-python-headless-3.4.18.65.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2ead52eddf27f3fd86bc9693994d2be64b0567f5770b68e968eb710fb967f085 -size 68068970 diff --git a/python 3.9.16/opencv-python-headless-4.4.0.46.zip b/python 3.9.16/opencv-python-headless-4.4.0.46.zip deleted file mode 100644 index eadafc7..0000000 --- a/python 3.9.16/opencv-python-headless-4.4.0.46.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ecec4f99c62252ee860a58f60ec185a83184ba467b04a444ad00ac8ec8192b37 -size 56938381 diff --git a/python 3.9.16/opencv-python-headless-4.5.5.64.zip b/python 3.9.16/opencv-python-headless-4.5.5.64.zip deleted file mode 100644 index c67260c..0000000 --- a/python 3.9.16/opencv-python-headless-4.5.5.64.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:040a68548460efef9c36a47d350d1681c854192f7e14572ad37d6cfe2d2a216a -size 68029113 diff --git a/python 3.9.16/opencv-python-headless-4.6.0.66.zip b/python 3.9.16/opencv-python-headless-4.6.0.66.zip deleted file mode 100644 index 67ff44c..0000000 --- a/python 3.9.16/opencv-python-headless-4.6.0.66.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7eeeeacebd8d61d3c74a205c006af0f381d919d66204a7c356e3f6489e5cf15c -size 68540705 diff --git a/python 3.9.16/opencv-python-headless-4.7.0.68.zip b/python 3.9.16/opencv-python-headless-4.7.0.68.zip deleted file mode 100644 index 841a978..0000000 --- a/python 3.9.16/opencv-python-headless-4.7.0.68.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c6432ca2c8b378f27afdc78d67fcfb656decdcedbb64bfb91efacd7db36e57a9 -size 69500343 diff --git a/scripts/build_layer.py b/scripts/build_layer.py new file mode 100644 index 0000000..6d0a843 --- /dev/null +++ b/scripts/build_layer.py @@ -0,0 +1,392 @@ +"""Build minimal OpenCV (headless) layers for AWS Lambda from source. + +Runs inside the official Lambda base image (public.ecr.aws/lambda/python:) +so glibc, compiler output, and bytecode all match the real runtime. Stdlib only. + +Stages (each writes into --artifacts): + opencv Build cv2 once per (version, profile, arch) with the Python Limited + API, so one binary serves every Python >= 3.12. + -> opencv---.tar.gz + numpy Build numpy without BLAS, once per (python, arch). + -> numpy--python-.tar.gz + package Combine every opencv tarball of this arch with this Python's numpy, + prune, precompile, smoke-test, and zip. + -> dist/opencv---python-.zip (+ .json) +""" + +import argparse +import compileall +import json +import os +import platform +import shutil +import subprocess +import sys +import sysconfig +import tarfile +import urllib.request +import zipfile +from pathlib import Path + +DEFAULT_NUMPY = "2.5.4" + +# OpenCV 5 renamed several modules. Names that do not exist in the source +# tree are dropped before configuring, so one list covers 4.x and 5.x. +MINIMAL = ["core", "imgproc", "imgcodecs"] +STANDARD = MINIMAL + [ + "features2d", "features", # 4.x, 5.x + "calib3d", "calib", "geometry", "stereo", + "flann", "objdetect", "photo", "video", "ml", +] +PROFILES = { + "minimal": MINIMAL, + "standard": STANDARD, + "dnn": STANDARD + ["dnn"], +} + +CMAKE_COMMON = { + "CMAKE_BUILD_TYPE": "Release", + "BUILD_SHARED_LIBS": "OFF", # everything linked statically into cv2.abi3.so + "OPENCV_FORCE_3RDPARTY_BUILD": "ON", + "BUILD_opencv_python3": "ON", + "BUILD_opencv_python2": "OFF", + "BUILD_JAVA": "OFF", + "BUILD_TESTS": "OFF", + "BUILD_PERF_TESTS": "OFF", + "BUILD_EXAMPLES": "OFF", + "BUILD_DOCS": "OFF", + "BUILD_opencv_apps": "OFF", + "OPENCV_GENERATE_PKGCONFIG": "OFF", + "INSTALL_PYTHON_EXAMPLES": "OFF", + "INSTALL_C_EXAMPLES": "OFF", + # Default Limited API version (3.6), same as the official abi3 wheels. + # Raising it to 3.12 breaks OpenCV's bindings: Py_INCREF and + # PyType_CheckExact become uncasted functions under that API level. + "PYTHON3_LIMITED_API": "ON", + # Image codecs: JPEG, PNG, WebP only, built from bundled sources. + "WITH_JPEG": "ON", "BUILD_JPEG": "ON", + "WITH_PNG": "ON", "BUILD_PNG": "ON", "BUILD_ZLIB": "ON", + "WITH_WEBP": "ON", "BUILD_WEBP": "ON", + "WITH_TIFF": "OFF", "WITH_OPENJPEG": "OFF", "WITH_JASPER": "OFF", + "WITH_OPENEXR": "OFF", "WITH_AVIF": "OFF", "WITH_JPEGXL": "OFF", + "WITH_IMGCODEC_HDR": "OFF", "WITH_IMGCODEC_SUNRASTER": "OFF", + "WITH_IMGCODEC_PXM": "OFF", "WITH_IMGCODEC_PFM": "OFF", + "WITH_GDAL": "OFF", "WITH_GDCM": "OFF", + # No video I/O, GUI, or accelerators. + "WITH_FFMPEG": "OFF", "WITH_GSTREAMER": "OFF", "WITH_V4L": "OFF", + "WITH_1394": "OFF", "WITH_OBSENSOR": "OFF", + "WITH_GTK": "OFF", "WITH_QT": "OFF", "WITH_OPENGL": "OFF", "WITH_VTK": "OFF", + "WITH_OPENCL": "OFF", "WITH_CUDA": "OFF", "WITH_VA": "OFF", "WITH_VA_INTEL": "OFF", + "WITH_IPP": "OFF", "WITH_TBB": "OFF", "WITH_OPENMP": "OFF", + "WITH_EIGEN": "OFF", "WITH_LAPACK": "OFF", "WITH_ITT": "OFF", "WITH_ADE": "OFF", + # Size: drop unused sections at link time. + "CMAKE_C_FLAGS": "-ffunction-sections -fdata-sections", + "CMAKE_CXX_FLAGS": "-ffunction-sections -fdata-sections", + "CMAKE_MODULE_LINKER_FLAGS": "-Wl,--gc-sections -Wl,--as-needed", + "CMAKE_SHARED_LINKER_FLAGS": "-Wl,--gc-sections -Wl,--as-needed", +} +CMAKE_ARCH = { + # Every Lambda x86_64 host supports AVX2, so make it the baseline and + # skip runtime dispatch copies (SSE4/AVX/AVX-512) entirely. + "x86_64": {"CPU_BASELINE": "AVX2", "CPU_DISPATCH": ""}, + "arm64": {}, +} +CMAKE_PROFILE = { + "minimal": {"WITH_PROTOBUF": "OFF", "WITH_FLATBUFFERS": "OFF", "WITH_QUIRC": "OFF"}, + "standard": {"WITH_PROTOBUF": "OFF", "WITH_FLATBUFFERS": "OFF"}, + "dnn": {"WITH_PROTOBUF": "ON", "BUILD_PROTOBUF": "ON"}, +} + +PRUNE_PATHS = [ + "bin", + "numpy/_core/include", + "numpy/_core/lib", + "numpy/random/lib", + "numpy/random/_examples", + "numpy/f2py", + "numpy/_pyinstaller", + "numpy/doc", +] +PRUNE_DIR_NAMES = {"tests", "__pycache__"} +PRUNE_SUFFIXES = {".pyi", ".pyx", ".pxd", ".h", ".c", ".a"} +PRUNE_FILE_NAMES = {"py.typed"} + +SMOKE_TEST = """ +import json, sys, cv2, numpy as np +img = np.zeros((64, 64, 3), np.uint8) +cv2.circle(img, (32, 32), 10, (255, 255, 255), -1) +img = cv2.GaussianBlur(cv2.resize(img, (128, 128)), (5, 5), 0) +gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) +for ext in (".jpg", ".png", ".webp"): + ok, buf = cv2.imencode(ext, img) + assert ok, ext + assert cv2.imdecode(buf, cv2.IMREAD_GRAYSCALE).shape == gray.shape, ext +profile = sys.argv[1] +if profile in ("standard", "dnn"): + cv2.ORB_create().detectAndCompute(gray, None) + cv2.findHomography + cv2.QRCodeDetector().detect(gray) +if profile == "dnn": + cv2.dnn.blobFromImage(img) +assert np.linalg.inv(np.eye(3)).trace() == 3 +print(json.dumps({"opencv": cv2.__version__, "numpy": np.__version__})) +""" + + +def arch() -> str: + return {"x86_64": "x86_64", "aarch64": "arm64"}[platform.machine()] + + +def py_version() -> str: + return f"{sys.version_info.major}.{sys.version_info.minor}" + + +def run(*cmd, **kw) -> None: + print("+", " ".join(map(str, cmd)), flush=True) + subprocess.run([str(c) for c in cmd], check=True, **kw) + + +def pip(*args) -> None: + run(sys.executable, "-m", "pip", "--disable-pip-version-check", "--no-cache-dir", + "--quiet", *args, env={**os.environ, "PIP_ROOT_USER_ACTION": "ignore"}) + + +def install_toolchain() -> None: + run("dnf", "install", "-y", "gcc", "gcc-c++", "binutils") + pip("install", "cmake", "ninja") + + +def strip_binaries(root: Path) -> None: + for so in root.rglob("*.so*"): + if so.is_file(): + run("strip", "--strip-all", so) + + +def dir_size(path: Path) -> int: + return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) + + +def mb(n: int) -> float: + return round(n / 2**20, 1) + + +def pack(src: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(dest, "w:gz") as tf: + for p in sorted(src.iterdir()): + tf.add(p, p.name) + + +def unpack(src: Path, dest: Path) -> None: + with tarfile.open(src) as tf: + tf.extractall(dest, filter="data") + + +# --------------------------------------------------------------------- opencv + + +def build_opencv(version: str, profile: str, out: Path) -> Path: + install_toolchain() + # numpy headers for the bindings; numpy >= 2 headers stay ABI-compatible. + pip("install", f"numpy=={DEFAULT_NUMPY}") + + src_root = Path("/tmp/opencv-src") + shutil.rmtree(src_root, ignore_errors=True) + tarball = Path(f"/tmp/opencv-{version}.tar.gz") + if not tarball.exists(): + url = f"https://github.com/opencv/opencv/archive/refs/tags/{version}.tar.gz" + print("+ download", url, flush=True) + urllib.request.urlretrieve(url, tarball) + unpack(tarball, src_root) + src = src_root / f"opencv-{version}" + + available = {p.name for p in (src / "modules").iterdir() if p.is_dir()} + modules = [m for m in PROFILES[profile] if m in available] + ["python3"] + + build, prefix = Path("/tmp/opencv-build"), Path("/tmp/opencv-install") + shutil.rmtree(build, ignore_errors=True) + shutil.rmtree(prefix, ignore_errors=True) + + import importlib + importlib.invalidate_caches() + import numpy # installed above + + defs = { + **CMAKE_COMMON, **CMAKE_ARCH[arch()], **CMAKE_PROFILE[profile], + "BUILD_LIST": ",".join(modules), + "CMAKE_INSTALL_PREFIX": str(prefix), + "OPENCV_PYTHON3_INSTALL_PATH": "python", + "PYTHON3_EXECUTABLE": sys.executable, + "PYTHON3_INCLUDE_DIR": sysconfig.get_paths()["include"], + "PYTHON3_NUMPY_INCLUDE_DIRS": numpy.get_include(), + } + run("cmake", "-S", src, "-B", build, "-G", "Ninja", + *[f"-D{k}={v}" for k, v in defs.items()]) + run("cmake", "--build", build) + run("cmake", "--install", build, "--strip") + + stage = Path("/tmp/opencv-stage") + shutil.rmtree(stage, ignore_errors=True) + shutil.copytree(prefix / "python", stage) + if not list(stage.glob("cv2/python-3/cv2*.so")): + sys.exit("python3 bindings were not built; check the cmake output above") + licenses = next(prefix.glob("share/licenses/opencv*"), None) + if licenses: + shutil.copytree(licenses, stage / "cv2" / "licenses") + strip_binaries(stage) + + dest = out / f"opencv-{version}-{profile}-{arch()}.tar.gz" + pack(stage, dest) + print(f"{dest.name}: {mb(dir_size(stage))} MB unpacked") + return dest + + +# ---------------------------------------------------------------------- numpy + + +def build_numpy(version: str, out: Path) -> Path: + install_toolchain() + stage = Path("/tmp/numpy-stage") + shutil.rmtree(stage, ignore_errors=True) + # No BLAS/LAPACK: numpy falls back to its bundled lapack_lite, which drops + # the ~27 MB OpenBLAS + gfortran runtime. Large matmul/linalg get slower. + pip("install", "--target", stage, "--no-deps", "--no-binary=numpy", + "--config-settings=setup-args=-Dallow-noblas=true", + "--config-settings=setup-args=-Dblas=none", + "--config-settings=setup-args=-Dlapack=none", + f"numpy=={version}") + strip_binaries(stage) + + dest = out / f"numpy-{version}-python{py_version()}-{arch()}.tar.gz" + pack(stage, dest) + print(f"{dest.name}: {mb(dir_size(stage))} MB unpacked") + return dest + + +# -------------------------------------------------------------------- package + + +def prune(site: Path) -> None: + for rel in PRUNE_PATHS: + p = site / rel + if p.is_dir(): + shutil.rmtree(p) + elif p.exists(): + p.unlink() + for p in sorted(site.rglob("*"), key=lambda x: len(x.parts), reverse=True): + if not p.exists(): + continue + if p.is_dir() and p.name in PRUNE_DIR_NAMES: + shutil.rmtree(p) + elif p.is_file() and (p.suffix in PRUNE_SUFFIXES or p.name in PRUNE_FILE_NAMES): + p.unlink() + + +def make_zip(site: Path, dest: Path) -> None: + """Deterministic zip: sorted entries, fixed timestamps, unix permissions kept.""" + dest.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as zf: + for f in sorted(p for p in site.rglob("*") if p.is_file()): + info = zipfile.ZipInfo(f.relative_to(site.parent).as_posix(), (1980, 1, 1, 0, 0, 0)) + info.external_attr = (f.stat().st_mode & 0o777 | 0o100000) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + zf.writestr(info, f.read_bytes(), compresslevel=9) + + +def package_one(cv_tar: Path, np_tar: Path, out: Path) -> dict: + # opencv---.tar.gz + _, cv_ver, profile, _ = cv_tar.name.removesuffix(".tar.gz").split("-") + work = Path("/tmp/layer") + site = work / "python" + shutil.rmtree(work, ignore_errors=True) + unpack(np_tar, site) + unpack(cv_tar, site) + + prune(site) + # /opt is read-only on Lambda, so Python can never write its own cache: + # ship bytecode. unchecked-hash avoids zip's 2-second mtime rounding + # invalidating every .pyc on cold start. + compileall.compile_dir( + str(site), quiet=1, workers=0, + invalidation_mode=compileall.py_compile.PycInvalidationMode.UNCHECKED_HASH, + ) + + env = {**os.environ, "PYTHONPATH": str(site), "PYTHONDONTWRITEBYTECODE": "1"} + result = subprocess.run( + [sys.executable, "-c", SMOKE_TEST, profile], + env=env, capture_output=True, text=True, + ) + if result.returncode != 0: + print(f"smoke test failed: {cv_tar.name}\n{result.stderr}", file=sys.stderr, flush=True) + raise RuntimeError(f"smoke test failed: {cv_tar.name}") + versions = json.loads(result.stdout) + + zip_path = out / f"opencv-{cv_ver}-{profile}-python{py_version()}-{arch()}.zip" + make_zip(site, zip_path) + info = { + "file": zip_path.name, + "profile": profile, + "python": py_version(), + "arch": arch(), + **versions, + "unzipped_mb": mb(dir_size(site)), + "zipped_mb": mb(zip_path.stat().st_size), + } + zip_path.with_suffix(".json").write_text(json.dumps(info, indent=2) + "\n") + print(json.dumps(info), flush=True) + return info + + +def package(artifacts: Path, out: Path) -> None: + np_tars = sorted(artifacts.glob(f"numpy-*-python{py_version()}-{arch()}.tar.gz")) + cv_tars = sorted(artifacts.glob(f"opencv-*-{arch()}.tar.gz")) + if len(np_tars) != 1 or not cv_tars: + sys.exit(f"need exactly one numpy and at least one opencv tarball, got {np_tars} {cv_tars}") + failed = [] + for cv_tar in cv_tars: + try: + package_one(cv_tar, np_tars[0], out) + except RuntimeError: + failed.append(cv_tar.name) + if failed: + sys.exit(f"failed: {', '.join(failed)}") + + +# ----------------------------------------------------------------------- main + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts", type=Path, default=Path("artifacts")) + parser.add_argument("--out", type=Path, default=Path("dist")) + sub = parser.add_subparsers(dest="stage", required=True) + + p = sub.add_parser("opencv") + p.add_argument("version") + p.add_argument("profile", choices=PROFILES) + + p = sub.add_parser("numpy") + p.add_argument("version", nargs="?", default=DEFAULT_NUMPY) + + sub.add_parser("package") + + p = sub.add_parser("all") + p.add_argument("version") + p.add_argument("profile", choices=PROFILES) + p.add_argument("--numpy", default=DEFAULT_NUMPY) + + args = parser.parse_args() + if args.stage == "opencv": + build_opencv(args.version, args.profile, args.artifacts) + elif args.stage == "numpy": + build_numpy(args.version, args.artifacts) + elif args.stage == "package": + package(args.artifacts, args.out) + else: + cv_tar = build_opencv(args.version, args.profile, args.artifacts) + np_tar = build_numpy(args.numpy, args.artifacts) + package_one(cv_tar, np_tar, args.out) + + +if __name__ == "__main__": + main()