From a6e583f1988e4e9a43d8e21204764f7c9c3ca472 Mon Sep 17 00:00:00 2001 From: prdai Date: Fri, 20 Nov 2026 14:14:06 +0530 Subject: [PATCH 1/2] feat(sce): replace file handoff with gRPC data access Co-authored-by: Codex --- .gitignore | 3 + CLAUDE.md | 2 +- README.md | 2 +- SECURITY.md | 3 +- benchmark/src/plot.py | 2 +- docs/e2e-secure-rads.txt | 42 +- proto/generate.sh | 74 +++ proto/sce/v1/envelope.proto | 11 + proto/sce/v1/records.proto | 28 + secure/csav/README.md | 2 +- secure/csav/registry-service/nsjail.cfg.tmpl | 20 +- secure/csav/registry-service/rootfs.go | 13 +- secure/pdm-service/src/handlers/records.ts | 6 +- secure/samples/README.md | 4 +- .../samples/audience-segments-rust/Cargo.lock | 617 +++++++++++++++++- .../samples/audience-segments-rust/Cargo.toml | 7 + .../samples/audience-segments-rust/README.md | 9 +- .../audience-segments-rust/src/gen/mod.rs | 6 + .../src/gen/sce/v1/sce.v1.rs | 35 + .../src/gen/sce/v1/sce.v1.tonic.rs | 395 +++++++++++ .../audience-segments-rust/src/main.rs | 150 ++++- secure/samples/health-risk-python/Dockerfile | 1 + secure/samples/health-risk-python/README.md | 17 +- secure/samples/health-risk-python/main.py | 51 +- .../samples/health-risk-python/pyproject.toml | 2 + .../health-risk-python/sce/__init__.py | 0 .../health-risk-python/sce/v1/__init__.py | 0 .../health-risk-python/sce/v1/envelope_pb2.py | 36 + .../sce/v1/envelope_pb2_grpc.py | 24 + .../health-risk-python/sce/v1/records_pb2.py | 45 ++ .../sce/v1/records_pb2_grpc.py | 145 ++++ secure/samples/health-risk-python/uv.lock | 74 ++- secure/samples/product-trends-node/Dockerfile | 9 +- secure/samples/product-trends-node/README.md | 17 +- .../gen/sce/v1/envelope_pb.d.ts | 41 ++ .../gen/sce/v1/envelope_pb.js | 19 + .../gen/sce/v1/records_pb.d.ts | 104 +++ .../gen/sce/v1/records_pb.js | 49 ++ secure/samples/product-trends-node/main.js | 74 ++- .../product-trends-node/package-lock.json | 347 ++++++++++ .../samples/product-trends-node/package.json | 4 + secure/samples/spend-analysis-go/Dockerfile | 5 +- secure/samples/spend-analysis-go/README.md | 17 +- .../gen/scev1/envelope.pb.go | 144 ++++ .../spend-analysis-go/gen/scev1/records.pb.go | 268 ++++++++ .../gen/scev1/records_grpc.pb.go | 171 +++++ secure/samples/spend-analysis-go/go.mod | 14 +- secure/samples/spend-analysis-go/go.sum | 18 + secure/samples/spend-analysis-go/main.go | 91 ++- .../trial-eligibility-python/Dockerfile | 1 + .../trial-eligibility-python/README.md | 17 +- .../samples/trial-eligibility-python/main.py | 54 +- .../trial-eligibility-python/pyproject.toml | 2 + .../trial-eligibility-python/sce/__init__.py | 0 .../sce/v1/__init__.py | 0 .../sce/v1/envelope_pb2.py | 36 + .../sce/v1/envelope_pb2_grpc.py | 24 + .../sce/v1/records_pb2.py | 45 ++ .../sce/v1/records_pb2_grpc.py | 145 ++++ .../samples/trial-eligibility-python/uv.lock | 74 ++- secure/sce-unit/Dockerfile | 3 +- secure/sce-unit/README.md | 17 +- secure/sce-unit/crypto.go | 44 +- secure/sce-unit/gen/scev1/envelope.pb.go | 144 ++++ secure/sce-unit/gen/scev1/records.pb.go | 268 ++++++++ secure/sce-unit/gen/scev1/records_grpc.pb.go | 171 +++++ secure/sce-unit/go.mod | 12 +- secure/sce-unit/go.sum | 24 +- secure/sce-unit/main.go | 130 ++-- secure/sce-unit/pdm.go | 5 +- secure/sce-unit/rpc.go | 155 +++++ seeding/bun.lock | 3 + seeding/gen/sce/v1/envelope_pb.ts | 44 ++ seeding/gen/sce/v1/records_pb.ts | 112 ++++ seeding/package.json | 1 + seeding/seed-secure-rads.ts | 17 +- 76 files changed, 4402 insertions(+), 364 deletions(-) create mode 100755 proto/generate.sh create mode 100644 proto/sce/v1/envelope.proto create mode 100644 proto/sce/v1/records.proto create mode 100644 secure/samples/audience-segments-rust/src/gen/mod.rs create mode 100644 secure/samples/audience-segments-rust/src/gen/sce/v1/sce.v1.rs create mode 100644 secure/samples/audience-segments-rust/src/gen/sce/v1/sce.v1.tonic.rs create mode 100644 secure/samples/health-risk-python/sce/__init__.py create mode 100644 secure/samples/health-risk-python/sce/v1/__init__.py create mode 100644 secure/samples/health-risk-python/sce/v1/envelope_pb2.py create mode 100644 secure/samples/health-risk-python/sce/v1/envelope_pb2_grpc.py create mode 100644 secure/samples/health-risk-python/sce/v1/records_pb2.py create mode 100644 secure/samples/health-risk-python/sce/v1/records_pb2_grpc.py create mode 100644 secure/samples/product-trends-node/gen/sce/v1/envelope_pb.d.ts create mode 100644 secure/samples/product-trends-node/gen/sce/v1/envelope_pb.js create mode 100644 secure/samples/product-trends-node/gen/sce/v1/records_pb.d.ts create mode 100644 secure/samples/product-trends-node/gen/sce/v1/records_pb.js create mode 100644 secure/samples/product-trends-node/package-lock.json create mode 100644 secure/samples/spend-analysis-go/gen/scev1/envelope.pb.go create mode 100644 secure/samples/spend-analysis-go/gen/scev1/records.pb.go create mode 100644 secure/samples/spend-analysis-go/gen/scev1/records_grpc.pb.go create mode 100644 secure/samples/spend-analysis-go/go.sum create mode 100644 secure/samples/trial-eligibility-python/sce/__init__.py create mode 100644 secure/samples/trial-eligibility-python/sce/v1/__init__.py create mode 100644 secure/samples/trial-eligibility-python/sce/v1/envelope_pb2.py create mode 100644 secure/samples/trial-eligibility-python/sce/v1/envelope_pb2_grpc.py create mode 100644 secure/samples/trial-eligibility-python/sce/v1/records_pb2.py create mode 100644 secure/samples/trial-eligibility-python/sce/v1/records_pb2_grpc.py create mode 100644 secure/sce-unit/gen/scev1/envelope.pb.go create mode 100644 secure/sce-unit/gen/scev1/records.pb.go create mode 100644 secure/sce-unit/gen/scev1/records_grpc.pb.go create mode 100644 secure/sce-unit/rpc.go create mode 100644 seeding/gen/sce/v1/envelope_pb.ts create mode 100644 seeding/gen/sce/v1/records_pb.ts diff --git a/.gitignore b/.gitignore index 7525f390..6eb53ed6 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ __pycache__/ # OpenCode local state .opencode/ + +# Rust build output +secure/samples/audience-segments-rust/target/ diff --git a/CLAUDE.md b/CLAUDE.md index f6de0c90..33a97ae3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ The human user is the technical lead. The agent is a supporting engineer that he ## Project Context -This repo is a PoC of the Secure Computing Environment (SCE), running on AWS Nitro Enclaves. It covers the end-to-end workflow: a build phase (Registry Service, part of the CSAV, turns a Data Controller's OCI image into a single EIF, layering the SCE Unit and nsjail on top, stores the artifact in S3, records `AlgorithmId` -> S3 path in DynamoDB) and an execution phase (enclave provisioning, remote attestation against the Verifier Service, data fetch from the Personal Data Management Service and KMS, computation over decrypted records files inside the enclave (`RECORDS_PATH` / `RESULT_PATH`, same contract as rads), async result delivery through SQS and a result-consumer Lambda, polled via `GET /run/{runId}`). See the architecture section in `README.md` and the docs it links before making assumptions about scope or design. +This repo is a PoC of the Secure Computing Environment (SCE), running on AWS Nitro Enclaves. It covers the end-to-end workflow: a build phase (Registry Service, part of the CSAV, turns a Data Controller's OCI image into a single EIF, layering the SCE Unit and nsjail on top, stores the artifact in S3, records `AlgorithmId` -> S3 path in DynamoDB) and an execution phase (enclave provisioning, remote attestation against the Verifier Service, data fetch from the Personal Data Management Service and KMS, computation over decrypted records streamed to the DC app over an AF_UNIX gRPC socket (`FetchRecords` / `SubmitResult`), async result delivery through SQS and a result-consumer Lambda, polled via `GET /run/{runId}`). See the architecture section in `README.md` and the docs it links before making assumptions about scope or design. ## Operating Rules diff --git a/README.md b/README.md index 3afd7842..eeb94a1a 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Every rads AWS resource is prefixed `rads-`; every secure-rads resource is prefi **Build phase (CSAV).** A Data Controller uploads an OCI image and assigns it an `algorithmId`. registry-service unpacks the image (daemonless: skopeo + umoci), layers the SCE Unit and nsjail on top, and builds a single Nitro Enclave Image File (EIF). It registers the EIF's PCR measurements against that `algorithmId` with the Verifier Service (over mTLS) and stores the EIF in S3, recording `algorithmId -> S3 path` in DynamoDB. -**Execution phase (SCE).** A caller POSTs an `algorithmId` to the API Gateway and Lambda execution-service control plane, which launches a per-run Nitro Enclave parent EC2 instance. Inside, the SCE Unit seeds enclave entropy from the NSM, attests to the Verifier Service, and - once attestation passes - fetches that algorithm's records from the PDM Service, gets the data key unwrapped by the KMS Service, decrypts them, and writes `records.json` for the jailed Data Controller app (`RECORDS_PATH` / `RESULT_PATH`, same file contract as rads). Results come back asynchronously: `/run` returns a `runId` immediately, the enclave host publishes the result to SQS, a consumer Lambda persists it, and the caller polls `GET /run/{runId}`. +**Execution phase (SCE).** A caller POSTs an `algorithmId` to the API Gateway and Lambda execution-service control plane, which launches a per-run Nitro Enclave parent EC2 instance. Inside, the SCE Unit seeds enclave entropy from the NSM, starts the jailed Data Controller app, and serves it over an AF_UNIX gRPC socket (`/data/sce.sock`). The app's `FetchRecords` call triggers attestation to the Verifier Service, the fetch from the PDM Service, and decryption with the data key unwrapped by the KMS Service. The app streams records, computes, and submits one result message that the SCE Unit converts back to JSON. Results come back asynchronously: `/run` returns a `runId` immediately, the enclave host publishes the result to SQS, a consumer Lambda persists it, and the caller polls `GET /run/{runId}`. Throughout both phases every component posts timestamped events to the telemetry-service, so a run's full timeline - build to execution - can be exported for benchmarking. The log-service records only the subject-access trail. Components also report selected, attributable AWS operations to the cost-service, keyed by the same `(AlgorithmId, runId)`, so their estimated costs can be summed per AWS service ([`docs/cost-service.md`](docs/cost-service.md)). Event-by-event plaintext flows: [`docs/e2e-rads.txt`](docs/e2e-rads.txt), [`docs/e2e-secure-rads.txt`](docs/e2e-secure-rads.txt). diff --git a/SECURITY.md b/SECURITY.md index bd80e3c0..06d731d1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -22,8 +22,7 @@ If the report involves a credential, revoke it first; do not send the secret its ## What is in scope - Attestation bypass: an enclave or process that should fail verification and does not. -- Trust boundary breaks: the Data Controller app reaching data outside its `RECORDS_PATH`, or - reading decrypted records after the run. +- Trust boundary breaks: the Data Controller app reaching data outside the records streamed over its `SCE_RPC_SOCKET`, or reading decrypted records after the run. - Enclave escape or unintended host access from `sce-unit` or the jailed app. - Secret exposure in the repository or in build artifacts. - IAM or resource policies that grant more than the documented flow requires. diff --git a/benchmark/src/plot.py b/benchmark/src/plot.py index 00915bc0..52b6b3e3 100644 --- a/benchmark/src/plot.py +++ b/benchmark/src/plot.py @@ -36,7 +36,7 @@ COST_COLORS = ["#1F4E79", "#4C9BE8", "#C0392B", "#6FBF73", "#F2C14E", "#8E44AD"] NOTE = ( "host provisioning = per-run instance boot + tooling install (size-independent); " - "data retrieval is fetch through records.json write (secure-rads includes decrypt).\n" + "data retrieval is fetch through decrypted record delivery (secure-rads includes decrypt).\n" "rads has no enclave, so enclave boot / sce unit / attestation are blank for it." ) diff --git a/docs/e2e-secure-rads.txt b/docs/e2e-secure-rads.txt index 75253731..58c7436c 100644 --- a/docs/e2e-secure-rads.txt +++ b/docs/e2e-secure-rads.txt @@ -70,26 +70,28 @@ Input: 3 emit enclave-started // no token yet, auth none 4 SEED-ENTROPY from NSM // no AWS cost 5 emit unit-ready // no token yet, auth none -6 challenge ← REQUEST-CHALLENGE(Verifier, a, runId) // single-use nonce, verifier API GW/Lambda and DynamoDB metered -7 token ← ATTEST(Verifier, runId, challenge) // NSM nonce ← challenge; claims the run so only one attestation succeeds; verifier API GW/Lambda and DynamoDB metered -8 emit attested -9 records ← FETCH-PDM(a, token, limit ← n) -10 cost apigateway:request + lambda:invocation + lambda:compute + dynamodb:read-capacity // PDM /records -11 emit records-fetched -12 ciphertext ← KMS-SERVICE-DECRYPT(token) // unwraps the data key under this enclave's measurements -13 cost apigateway:request + lambda:invocation + lambda:compute + dynamodb:read-capacity // kms-service /decrypt -14 DEK ← UNWRAP(ciphertext, enclaveKey) // KMS request itself is excluded from cost -15 plaintext ← DECRYPT(records, DEK) -16 emit records-decrypted -17 write {algorithmId, records} to /data/records.json -18 emit records-stored-locally -19 START nsjail // RECORDS_PATH, RESULT_PATH -20 wait for nsjail exit -21 output ← read /data/result.json -22 emit dc-app-completed // on success, else run-failed -23 STREAM output to parent // vsock :5000 -24 emit output-sent -25 // on failure: emit run-failed +6 START nsjail // SCE_RPC_SOCKET=/data/sce.sock, ALGORITHM_ID +7 wait for app FetchRecords RPC // first call triggers the steps below; retries reuse the outcome +8 challenge ← REQUEST-CHALLENGE(Verifier, a, runId) // single-use nonce, verifier API GW/Lambda and DynamoDB metered +9 token ← ATTEST(Verifier, runId, challenge) // NSM nonce ← challenge; claims the run so only one attestation succeeds; verifier API GW/Lambda and DynamoDB metered +10 emit attested +11 records ← FETCH-PDM(a, token, limit ← n) +12 cost apigateway:request + lambda:invocation + lambda:compute + dynamodb:read-capacity // PDM /records +13 emit records-fetched +14 ciphertext ← KMS-SERVICE-DECRYPT(token) // unwraps the data key under this enclave's measurements +15 cost apigateway:request + lambda:invocation + lambda:compute + dynamodb:read-capacity // kms-service /decrypt +16 DEK ← UNWRAP(ciphertext, enclaveKey) // KMS request itself is excluded from cost +17 plaintext ← DECRYPT(records, DEK) // RecordEnvelope -> sce.v1.Record +18 emit records-decrypted +19 emit records-stored-locally +20 STREAM records to app // gRPC FetchRecords, one sce.v1.Record per message +21 wait for app SubmitResult RPC // google.protobuf.Struct +22 wait for nsjail exit +23 emit dc-app-completed // on success, else run-failed +24 output ← JSON(result) +25 STREAM output to parent // vsock :5000 +26 emit output-sent +27 // on failure: emit run-failed diff --git a/proto/generate.sh b/proto/generate.sh new file mode 100755 index 00000000..c9369deb --- /dev/null +++ b/proto/generate.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Regenerates the checked-in gRPC stubs from proto/. Run after editing any .proto. +# Each language gets its own output tree; the .proto files stay the single source of truth. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +proto_dir="$repo_root/proto" +wkt_include="/usr/include" + +protoc_version="$(protoc --version)" +echo "using $protoc_version" + +gen_go() { + local module="$1" out="$2" go_import="$3" + mkdir -p "$out" + protoc -I "$proto_dir" -I "$wkt_include" \ + --plugin=protoc-gen-go="${PROTOC_GEN_GO:-protoc-gen-go}" \ + --plugin=protoc-gen-go-grpc="${PROTOC_GEN_GO_GRPC:-protoc-gen-go-grpc}" \ + --go_out="$out" --go_opt=module="$module" \ + --go_opt=Msce/v1/records.proto="$go_import" \ + --go_opt=Msce/v1/envelope.proto="$go_import" \ + --go-grpc_out="$out" --go-grpc_opt=module="$module" \ + --go-grpc_opt=Msce/v1/records.proto="$go_import" \ + --go-grpc_opt=Msce/v1/envelope.proto="$go_import" \ + sce/v1/records.proto sce/v1/envelope.proto +} + +# sce-unit (Go server) +gen_go "sce-unit" "$repo_root/secure/sce-unit" "sce-unit/gen/scev1" + +# spend-analysis (Go client) +gen_go "spend-analysis" "$repo_root/secure/samples/spend-analysis-go" "spend-analysis/gen/scev1" + +# Python clients: generate at the app root so `from sce.v1 import ...` resolves with the app dir on sys.path. +for app in health-risk-python trial-eligibility-python; do + out="$repo_root/secure/samples/$app" + mkdir -p "$out/sce/v1" + python -m grpc_tools.protoc -I "$proto_dir" -I "$wkt_include" \ + --python_out="$out" --grpc_python_out="$out" \ + sce/v1/records.proto sce/v1/envelope.proto + for pkg in sce sce/v1; do + touch "$out/$pkg/__init__.py" + done +done +rm -f "$repo_root/secure/samples/health-risk-python"/{records,envelope}_pb2*.py \ + "$repo_root/secure/samples/trial-eligibility-python"/{records,envelope}_pb2*.py + +# Seeding (TypeScript / protobuf-es) +mkdir -p "$repo_root/seeding/gen" +"${PROTOC_GEN_ES:-protoc-gen-es}" --version >/dev/null 2>&1 || true +protoc -I "$proto_dir" -I "$wkt_include" \ + --plugin=protoc-gen-es="${PROTOC_GEN_ES:-protoc-gen-es}" \ + --es_out="$repo_root/seeding/gen" --es_opt=target=ts \ + sce/v1/records.proto sce/v1/envelope.proto + +# Node client (protobuf-es, JS + d.ts so the sample stays plain JS) +node_out="$repo_root/secure/samples/product-trends-node/gen" +mkdir -p "$node_out" +protoc -I "$proto_dir" -I "$wkt_include" \ + --plugin=protoc-gen-es="${PROTOC_GEN_ES:-protoc-gen-es}" \ + --es_out="$node_out" --es_opt=target=js+dts \ + sce/v1/records.proto sce/v1/envelope.proto + +# Rust client (prost + tonic) +rust_out="$repo_root/secure/samples/audience-segments-rust/src/gen" +mkdir -p "$rust_out/sce/v1" +protoc -I "$proto_dir" -I "$wkt_include" \ + --plugin=protoc-gen-prost="${PROTOC_GEN_PROST:-protoc-gen-prost}" \ + --plugin=protoc-gen-tonic="${PROTOC_GEN_TONIC:-protoc-gen-tonic}" \ + --prost_out="$rust_out" \ + --tonic_out="$rust_out" \ + sce/v1/records.proto sce/v1/envelope.proto + +echo "generated stubs" diff --git a/proto/sce/v1/envelope.proto b/proto/sce/v1/envelope.proto new file mode 100644 index 00000000..2f5d21c6 --- /dev/null +++ b/proto/sce/v1/envelope.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package sce.v1; + +// RecordEnvelope is what pdm-service stores in DynamoDB Data. ciphertext is AES-256-GCM over a serialized +// sce.v1.Record; nonce is the 96-bit GCM nonce, and the auth tag is appended to ciphertext (Go GCM layout). +message RecordEnvelope { + uint32 version = 1; + bytes nonce = 2; + bytes ciphertext = 3; +} diff --git a/proto/sce/v1/records.proto b/proto/sce/v1/records.proto new file mode 100644 index 00000000..3e0d64d4 --- /dev/null +++ b/proto/sce/v1/records.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +package sce.v1; + +import "google/protobuf/struct.proto"; + +// Record is one personal-data record handed to the DC application. record_id stays separate so the DC app +// never sees data subject ids; data is schemaless per dataset, so it travels as a Struct. +message Record { + string record_id = 1; + google.protobuf.Struct data = 2; +} + +message FetchRecordsRequest {} + +message SubmitResultRequest { + google.protobuf.Struct result = 1; +} + +message SubmitResultResponse {} + +// DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. +service DataAccess { + // Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. + rpc FetchRecords(FetchRecordsRequest) returns (stream Record); + // The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. + rpc SubmitResult(SubmitResultRequest) returns (SubmitResultResponse); +} diff --git a/secure/csav/README.md b/secure/csav/README.md index f5043eb3..ca8affcb 100644 --- a/secure/csav/README.md +++ b/secure/csav/README.md @@ -25,7 +25,7 @@ flowchart TD C --> G["opt/controller/rootfs/ (the DC app)"] ``` -- **nsjail**: enforces the privilege boundary at runtime — DC algorithm gets its rootfs and `RECORDS_PATH` / `RESULT_PATH` files under `/data`, nothing else. +- **nsjail**: enforces the privilege boundary at runtime — the DC algorithm gets its rootfs, a `/data` scratch dir, and the AF_UNIX gRPC socket sce-unit serves at `/data/sce.sock`, nothing else. - **Trust material**: public key pin or CA cert injected into the EIF at build time, so the enclave can verify the Verifier Service, Personal Data Management Service, KMS Service, and Log Service later. - **Storage**: final EIF goes to S3, `AlgorithmId` -> S3 path goes to DynamoDB. Registry Service also generates PCR values and sends them to the Verifier Service. - **KMS keys**: one PCR-bound key per algorithm, created on first build and reused on rebuild (its policy is rewritten to the new measurements), so a build never orphans the previous key. The policy grants `Decrypt` to the kms-service role under `kms:RecipientAttestation` conditions; the enclave and its host never decrypt directly. diff --git a/secure/csav/registry-service/nsjail.cfg.tmpl b/secure/csav/registry-service/nsjail.cfg.tmpl index 6ac61d04..d3b33ed6 100644 --- a/secure/csav/registry-service/nsjail.cfg.tmpl +++ b/secure/csav/registry-service/nsjail.cfg.tmpl @@ -2,9 +2,8 @@ # exec_bin is generated per-build from the uploaded image's own # ENTRYPOINT/CMD - no universal binary path across arbitrary DC images. # -# Records/results live under /data in the DC rootfs (sce-unit writes -# records.json there before start; the app writes result.json). Same -# RECORDS_PATH / RESULT_PATH contract as rads. +# The DC app pulls records and submits its result over the gRPC socket +# sce-unit serves at /data/sce.sock inside the jail (see sce-unit/rpc.go). name: "data-controller-application-jail" @@ -16,13 +15,13 @@ mount_proc: true # nsjail strips env vars by default; PATH here is sourced from the DC's own image (see pathFromEnv in oci.go). envar: "PATH={{.PathEnv}}" -envar: "RECORDS_PATH=/data/records.json" -envar: "RESULT_PATH=/data/result.json" -# Without HOME, glibc NSS calls getpwuid for ~ expansion, which opens an AF_UNIX socket the jail now blocks. +envar: "SCE_RPC_SOCKET=/data/sce.sock" +envar: "ALGORITHM_ID={{.AlgorithmId}}" +# Without HOME, glibc NSS calls getpwuid for ~ expansion. envar: "HOME=/data" # rlimits left deliberately loose — lockdown is the seccomp allowlist below (see csav/README.md). -# fsize raised so large records.json dumps (size sweeps) fit on disk inside the jail. +# fsize raised so app scratch files (size sweeps) fit on disk inside the jail. rlimit_as: 2048 rlimit_cpu: 3600 rlimit_fsize: 8192 @@ -47,8 +46,8 @@ mount { nodev: true } -# Allowlist for the shipped samples: file I/O and process control only. No socket -# syscalls are allowed; the DC app only reads/writes files under /data. +# Allowlist for the shipped samples: file I/O, process control, and AF_UNIX +# sockets for the sce-unit gRPC channel. The jail stays loopback-free. seccomp_string: "ALLOW {" seccomp_string: " read, write, close, lseek, pread64, pwrite64, readv, writev," seccomp_string: " openat, openat2, newfstatat, newfstat, statx, getdents64, faccessat, faccessat2," @@ -64,6 +63,9 @@ seccomp_string: " getcwd, ioctl, fcntl, pipe2, dup, dup3," seccomp_string: " gettimeofday, clock_gettime, clock_nanosleep, nanosleep, getrandom," seccomp_string: " prctl, prlimit64, getrlimit, newuname, sysinfo, sched_yield, sched_getaffinity," seccomp_string: " epoll_create1, epoll_ctl, epoll_pwait, epoll_pwait2, ppoll, pselect6," +seccomp_string: " socket, socketpair, connect, bind, listen, accept, accept4," +seccomp_string: " sendto, recvfrom, sendmsg, recvmsg, shutdown, getsockopt, setsockopt," +seccomp_string: " getpeername, getsockname," seccomp_string: " eventfd2" seccomp_string: "}" seccomp_string: "DEFAULT KILL_PROCESS" diff --git a/secure/csav/registry-service/rootfs.go b/secure/csav/registry-service/rootfs.go index 80196fda..9efe35db 100644 --- a/secure/csav/registry-service/rootfs.go +++ b/secure/csav/registry-service/rootfs.go @@ -72,7 +72,7 @@ func composeFinalRootfs(bundleDir, execPath, pathEnv string, execArgs []string, if err := os.MkdirAll(cfgDir, 0o755); err != nil { return err } - if err := writeNsjailCfg(filepath.Join(cfgDir, "nsjail.cfg"), execPath, pathEnv, execArgs); err != nil { + if err := writeNsjailCfg(filepath.Join(cfgDir, "nsjail.cfg"), execPath, pathEnv, execArgs, cfg.AlgorithmId); err != nil { return err } if err := writeRunConfig(filepath.Join(cfgDir, "run-config.json"), cfg); err != nil { @@ -85,7 +85,7 @@ func composeFinalRootfs(bundleDir, execPath, pathEnv string, execArgs []string, return os.Rename(tmpFinal, dcRootfs) } -func writeNsjailCfg(path, execPath, pathEnv string, args []string) error { +func writeNsjailCfg(path, execPath, pathEnv string, args []string, algorithmID string) error { tmpl, err := template.New("nsjail.cfg").Parse(nsjailCfgTmpl) if err != nil { return err @@ -98,10 +98,11 @@ func writeNsjailCfg(path, execPath, pathEnv string, args []string) error { defer out.Close() return tmpl.Execute(out, struct { - Path string - PathEnv string - Args []string - }{Path: execPath, PathEnv: pathEnv, Args: args}) + Path string + PathEnv string + Args []string + AlgorithmId string + }{Path: execPath, PathEnv: pathEnv, Args: args, AlgorithmId: algorithmID}) } func writeRunConfig(path string, cfg runConfig) error { diff --git a/secure/pdm-service/src/handlers/records.ts b/secure/pdm-service/src/handlers/records.ts index f92439be..d9e45166 100644 --- a/secure/pdm-service/src/handlers/records.ts +++ b/secure/pdm-service/src/handlers/records.ts @@ -84,7 +84,11 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { // A standalone list aligned with `records`; the enclave assembles the DSAR trail from it and posts after a // successful DC app run. Record objects stay {recordId, data}, so the DC app never sees subject ids. const items = result.Items ?? []; - const records = items.map((item) => ({ recordId: item.RecordId, data: item.Data })); + const records = items.map((item) => ({ + recordId: item.RecordId, + // Data is a DynamoDB Binary holding a serialized RecordEnvelope; JSON transport needs it base64. + data: Buffer.from(item.Data as Uint8Array).toString("base64"), + })); const dataSubjectIds = items.map((item) => (typeof item.DataSubjectId === "string" ? item.DataSubjectId : "")); const nextCursor = result.LastEvaluatedKey ? Buffer.from(JSON.stringify(result.LastEvaluatedKey)).toString("base64url") diff --git a/secure/samples/README.md b/secure/samples/README.md index 1fffeb4e..26378845 100644 --- a/secure/samples/README.md +++ b/secure/samples/README.md @@ -1,6 +1,6 @@ # Sample Data Controller apps -Reference Data Controller (DC) images for the demo. Each is a self-contained app in a different language that reads the records file at `RECORDS_PATH` (JSON written by sce-unit after decrypt), aggregates, and writes one JSON result to `RESULT_PATH`. Same file contract as rads. All ignore the PII block on each record; the computation only touches the context fields. +Reference Data Controller (DC) images for the demo. Each is a self-contained app in a different language that calls sce-unit over the gRPC socket at `SCE_RPC_SOCKET` (`/data/sce.sock`), streams decrypted records with `FetchRecords`, aggregates them, and submits one result with `SubmitResult`. The protobuf contract lives in [`proto/sce/v1`](../../proto/sce/v1); generated stubs are checked in per app. All ignore the PII block on each record; the computation only touches the context fields. Each app is pinned to the fixed `algorithmId` its dataset is seeded under (see `seeding/config.ts`), so the whole demo - seed, build, run - lines up and is repeatable. The root `Makefile` drives all five (`make secure-rads-all`). @@ -20,4 +20,4 @@ Each app's own README documents its exact input shape and the report it returns. docker build --platform linux/arm64 -t :local secure/samples/ ``` -The images are arm64 to match the Graviton enclave hosts. Inside the enclave, nsjail sets `RECORDS_PATH=/data/records.json` and `RESULT_PATH=/data/result.json`. +The images are arm64 to match the Graviton enclave hosts. Inside the enclave, nsjail sets `SCE_RPC_SOCKET=/data/sce.sock` and `ALGORITHM_ID`; sce-unit serves the gRPC API on that socket. diff --git a/secure/samples/audience-segments-rust/Cargo.lock b/secure/samples/audience-segments-rust/Cargo.lock index c070e50a..129000cd 100644 --- a/secure/samples/audience-segments-rust/Cargo.lock +++ b/secure/samples/audience-segments-rust/Cargo.lock @@ -2,12 +2,299 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "audience-segments" version = "0.1.0" dependencies = [ + "hyper-util", + "prost", + "prost-types", "serde", "serde_json", + "tokio", + "tonic", + "tonic-prost", + "tower", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bytes" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "776dfebd015e2d8732c684a2c71003674ccb49457b539b9e82151ba38e96dd7c" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", ] [[package]] @@ -16,12 +303,79 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mime" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5848f5a9e8117cf94f2f90faac24a87ff81600ff8628c1e3deaab572e76bfa1" + +[[package]] +name = "mio" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20521a903f02b0a573cd6840e9d0a5a0cc043d361fe924092cd70f8556cc9c4a" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6c2123f4708bbd422630a6dae7c03f678b1ce1f1c99e4c4b3beac69c8174cc9" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "358938fd6df944c1e44bb328b60fb275f13c804aac47ba26a1165d45a3f87756" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c938015719552d665fd0553b90d96cd7ff58e83e4c061ca99e6bb78865a72752" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -31,6 +385,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef0b569d1d907ab3a28596aee62cf1f61dacb0529a4bedcd8e7460e2d5228b4" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad9c5423459fa2f729b8d9235fab810aa29628b171467744fc86d21460ad70d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29a7c4c7a84cef41a53d338a2d10d4f1b043ea5c3dde26db22ee2c9c1b1c75b" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.47" @@ -67,7 +453,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.4", ] [[package]] @@ -83,6 +469,39 @@ dependencies = [ "zmij", ] +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3244542535d5de959d90ec55d1b5ff0fd2bb7b1310223907d5f11932847f7b59" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "3.0.4" @@ -94,12 +513,208 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2cdfc673aabd30fe8fee92e65837b644c37c5d47f0f97ae6d5e0f1237d15e1b" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba464054e2226f182a34489b8b079bf4c6e3e7ab5fd7fc30e024f24aa92a484" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/secure/samples/audience-segments-rust/Cargo.toml b/secure/samples/audience-segments-rust/Cargo.toml index 68704f85..eac8ef5e 100644 --- a/secure/samples/audience-segments-rust/Cargo.toml +++ b/secure/samples/audience-segments-rust/Cargo.toml @@ -4,5 +4,12 @@ version = "0.1.0" edition = "2021" [dependencies] +prost = "0.14" +prost-types = "0.14" serde = { version = "1", features = ["derive"] } serde_json = "1" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net"] } +tonic = { version = "0.14", features = ["transport"] } +tonic-prost = "0.14" +tower = { version = "0.5", features = ["util"] } +hyper-util = { version = "0.1", features = ["tokio"] } diff --git a/secure/samples/audience-segments-rust/README.md b/secure/samples/audience-segments-rust/README.md index 41357092..886ea091 100644 --- a/secure/samples/audience-segments-rust/README.md +++ b/secure/samples/audience-segments-rust/README.md @@ -1,8 +1,8 @@ # audience-segments-rust -Sample Data Controller app (Rust, serde_json only). Reads social-media records from -`RECORDS_PATH` (per-user PII, interests, connections, and 12 weeks of engagement) and -writes an ad-audience segmentation to `RESULT_PATH`: top 10 interests by user count with +Sample Data Controller app (Rust; tonic + serde_json). Streams social-media records over +the gRPC socket at `SCE_RPC_SOCKET` (per-user PII, interests, connections, and 12 weeks of +engagement) and submits an ad-audience segmentation: top 10 interests by user count with mean weekly active minutes, engagement tiers by mean weekly minutes (low < 30, medium 30-120, high > 120), and mean connection count. PII fields are ignored by the computation. @@ -11,3 +11,6 @@ Build: ```sh docker build --platform linux/arm64 -t audience-segments-rust secure/samples/audience-segments-rust ``` + +Inside the enclave nsjail sets `SCE_RPC_SOCKET=/data/sce.sock` and `ALGORITHM_ID`; +sce-unit serves the gRPC API on that socket. diff --git a/secure/samples/audience-segments-rust/src/gen/mod.rs b/secure/samples/audience-segments-rust/src/gen/mod.rs new file mode 100644 index 00000000..a675b8ad --- /dev/null +++ b/secure/samples/audience-segments-rust/src/gen/mod.rs @@ -0,0 +1,6 @@ +#[allow(dead_code)] +pub mod sce { + pub mod v1 { + include!("sce/v1/sce.v1.rs"); + } +} diff --git a/secure/samples/audience-segments-rust/src/gen/sce/v1/sce.v1.rs b/secure/samples/audience-segments-rust/src/gen/sce/v1/sce.v1.rs new file mode 100644 index 00000000..b131d706 --- /dev/null +++ b/secure/samples/audience-segments-rust/src/gen/sce/v1/sce.v1.rs @@ -0,0 +1,35 @@ +// @generated +// This file is @generated by prost-build. +/// Record is one personal-data record handed to the DC application. record_id stays separate so the DC app +/// never sees data subject ids; data is schemaless per dataset, so it travels as a Struct. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Record { + #[prost(string, tag = "1")] + pub record_id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub data: ::core::option::Option<::prost_types::Struct>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct FetchRecordsRequest { +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SubmitResultRequest { + #[prost(message, optional, tag = "1")] + pub result: ::core::option::Option<::prost_types::Struct>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SubmitResultResponse { +} +/// RecordEnvelope is what pdm-service stores in DynamoDB Data. ciphertext is AES-256-GCM over a serialized +/// sce.v1.Record; nonce is the 96-bit GCM nonce, and the auth tag is appended to ciphertext (Go GCM layout). +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RecordEnvelope { + #[prost(uint32, tag = "1")] + pub version: u32, + #[prost(bytes = "vec", tag = "2")] + pub nonce: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "3")] + pub ciphertext: ::prost::alloc::vec::Vec, +} +include!("sce.v1.tonic.rs"); +// @@protoc_insertion_point(module) diff --git a/secure/samples/audience-segments-rust/src/gen/sce/v1/sce.v1.tonic.rs b/secure/samples/audience-segments-rust/src/gen/sce/v1/sce.v1.tonic.rs new file mode 100644 index 00000000..4c9ca7f7 --- /dev/null +++ b/secure/samples/audience-segments-rust/src/gen/sce/v1/sce.v1.tonic.rs @@ -0,0 +1,395 @@ +// @generated +/// Generated client implementations. +pub mod data_access_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + /** DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. +*/ + #[derive(Debug, Clone)] + pub struct DataAccessClient { + inner: tonic::client::Grpc, + } + impl DataAccessClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl DataAccessClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> DataAccessClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + DataAccessClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /** Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. +*/ + pub async fn fetch_records( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sce.v1.DataAccess/FetchRecords", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sce.v1.DataAccess", "FetchRecords")); + self.inner.server_streaming(req, path, codec).await + } + /** The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. +*/ + pub async fn submit_result( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sce.v1.DataAccess/SubmitResult", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sce.v1.DataAccess", "SubmitResult")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod data_access_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with DataAccessServer. + #[async_trait] + pub trait DataAccess: std::marker::Send + std::marker::Sync + 'static { + /// Server streaming response type for the FetchRecords method. + type FetchRecordsStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result, + > + + std::marker::Send + + 'static; + /** Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. +*/ + async fn fetch_records( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + /** The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. +*/ + async fn submit_result( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + /** DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. +*/ + #[derive(Debug)] + pub struct DataAccessServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl DataAccessServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for DataAccessServer + where + T: DataAccess, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/sce.v1.DataAccess/FetchRecords" => { + #[allow(non_camel_case_types)] + struct FetchRecordsSvc(pub Arc); + impl< + T: DataAccess, + > tonic::server::ServerStreamingService + for FetchRecordsSvc { + type Response = super::Record; + type ResponseStream = T::FetchRecordsStream; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::fetch_records(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = FetchRecordsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sce.v1.DataAccess/SubmitResult" => { + #[allow(non_camel_case_types)] + struct SubmitResultSvc(pub Arc); + impl< + T: DataAccess, + > tonic::server::UnaryService + for SubmitResultSvc { + type Response = super::SubmitResultResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::submit_result(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = SubmitResultSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + let mut response = http::Response::new( + tonic::body::Body::default(), + ); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } + } + } + } + impl Clone for DataAccessServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sce.v1.DataAccess"; + impl tonic::server::NamedService for DataAccessServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/secure/samples/audience-segments-rust/src/main.rs b/secure/samples/audience-segments-rust/src/main.rs index c5b751a0..2f0685b8 100644 --- a/secure/samples/audience-segments-rust/src/main.rs +++ b/secure/samples/audience-segments-rust/src/main.rs @@ -1,5 +1,11 @@ +mod gen; + +use gen::sce::v1::data_access_client::DataAccessClient; +use gen::sce::v1::{FetchRecordsRequest, SubmitResultRequest}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use tonic::transport::{Endpoint, Uri}; +use tower::service_fn; #[derive(Deserialize)] #[serde(rename_all = "camelCase")] @@ -28,13 +34,6 @@ struct Record { data: Data, } -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -struct RecordsFile { - algorithm_id: String, - records: Vec, -} - #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct Segment { @@ -59,14 +58,20 @@ fn round2(v: f64) -> f64 { fn analyze(algorithm_id: &str, records: &[Record]) -> Output { let mut per_interest: HashMap<&str, (usize, f64)> = HashMap::new(); - let mut tiers: HashMap = - [("low", 0), ("medium", 0), ("high", 0)].map(|(k, v)| (k.to_string(), v)).into(); + let mut tiers: HashMap = [("low", 0), ("medium", 0), ("high", 0)] + .map(|(k, v)| (k.to_string(), v)) + .into(); let mut connections_sum = 0usize; for rec in records { let weeks = rec.data.engagement.len().max(1) as f64; - let mean_minutes = - rec.data.engagement.iter().map(|e| e.minutes_active).sum::() / weeks; + let mean_minutes = rec + .data + .engagement + .iter() + .map(|e| e.minutes_active) + .sum::() + / weeks; let tier = if mean_minutes < 30.0 { "low" } else if mean_minutes <= 120.0 { @@ -104,32 +109,109 @@ fn analyze(algorithm_id: &str, records: &[Record]) -> Output { } } -fn load_records(path: &str) -> Result<(String, Vec), String> { - let body = std::fs::read_to_string(path).map_err(|e| format!("read {path} failed: {e}"))?; - let file: RecordsFile = - serde_json::from_str(&body).map_err(|e| format!("parse {path} failed: {e}"))?; - Ok((file.algorithm_id, file.records)) -} - -fn write_result(result: &Output) -> Result<(), String> { - let out = - serde_json::to_string(result).map_err(|e| format!("serialize result failed: {e}"))?; - match std::env::var("RESULT_PATH") { - Ok(path) if !path.is_empty() => { - std::fs::write(&path, out).map_err(|e| format!("write {path} failed: {e}")) - } - _ => { - println!("{out}"); - Ok(()) +fn value_to_json(value: &prost_types::Value) -> serde_json::Value { + use prost_types::value::Kind; + match value.kind.as_ref() { + Some(Kind::NullValue(_)) => serde_json::Value::Null, + Some(Kind::NumberValue(n)) => serde_json::Number::from_f64(*n) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null), + Some(Kind::StringValue(s)) => serde_json::Value::String(s.clone()), + Some(Kind::BoolValue(b)) => serde_json::Value::Bool(*b), + Some(Kind::StructValue(s)) => struct_to_json(s), + Some(Kind::ListValue(l)) => { + serde_json::Value::Array(l.values.iter().map(value_to_json).collect()) } + None => serde_json::Value::Null, } } -fn main() { - let records_path = std::env::var("RECORDS_PATH").expect("RECORDS_PATH is required"); +fn struct_to_json(value: &prost_types::Struct) -> serde_json::Value { + serde_json::Value::Object( + value + .fields + .iter() + .map(|(k, v)| (k.clone(), value_to_json(v))) + .collect(), + ) +} - let (algorithm_id, records) = load_records(&records_path).expect("load records"); - eprintln!("loaded {} records from {records_path}", records.len()); +fn json_to_value(value: &serde_json::Value) -> prost_types::Value { + use prost_types::value::Kind; + let kind = match value { + serde_json::Value::Null => Kind::NullValue(0), + serde_json::Value::Bool(b) => Kind::BoolValue(*b), + serde_json::Value::Number(n) => Kind::NumberValue(n.as_f64().unwrap_or(0.0)), + serde_json::Value::String(s) => Kind::StringValue(s.clone()), + serde_json::Value::Array(items) => Kind::ListValue(prost_types::ListValue { + values: items.iter().map(json_to_value).collect(), + }), + serde_json::Value::Object(_) => Kind::StructValue(json_to_struct(value)), + }; + prost_types::Value { kind: Some(kind) } +} + +fn json_to_struct(value: &serde_json::Value) -> prost_types::Struct { + let fields = match value { + serde_json::Value::Object(map) => map + .iter() + .map(|(k, v)| (k.clone(), json_to_value(v))) + .collect(), + _ => Default::default(), + }; + prost_types::Struct { fields } +} + +async fn channel(socket_path: &str) -> Result { + let path = socket_path.to_string(); + Endpoint::try_from("http://[::]:50051") + .map_err(|e| format!("build endpoint: {e}"))? + .connect_with_connector(service_fn(move |_: Uri| { + let path = path.clone(); + async move { + let stream = tokio::net::UnixStream::connect(path).await?; + Ok::<_, std::io::Error>(hyper_util::rt::TokioIo::new(stream)) + } + })) + .await + .map_err(|e| format!("connect {socket_path}: {e}")) +} - write_result(&analyze(&algorithm_id, &records)).expect("write result"); -} \ No newline at end of file +#[tokio::main] +async fn main() { + let socket_path = std::env::var("SCE_RPC_SOCKET").expect("SCE_RPC_SOCKET is required"); + let algorithm_id = std::env::var("ALGORITHM_ID").expect("ALGORITHM_ID is required"); + + let channel = channel(&socket_path).await.expect("connect to sce-unit"); + let mut client = DataAccessClient::new(channel); + + let mut stream = client + .fetch_records(FetchRecordsRequest {}) + .await + .expect("fetch records") + .into_inner(); + + let mut records: Vec = Vec::new(); + while let Some(record) = stream.message().await.expect("receive record") { + let data = record + .data + .as_ref() + .map(struct_to_json) + .unwrap_or(serde_json::Value::Object(Default::default())); + let data: Data = serde_json::from_value(data).expect("decode record"); + records.push(Record { + record_id: record.record_id, + data, + }); + } + eprintln!("loaded {} records over {socket_path}", records.len()); + + let result = analyze(&algorithm_id, &records); + let payload = serde_json::to_value(&result).expect("serialize result"); + client + .submit_result(SubmitResultRequest { + result: Some(json_to_struct(&payload)), + }) + .await + .expect("submit result"); +} diff --git a/secure/samples/health-risk-python/Dockerfile b/secure/samples/health-risk-python/Dockerfile index 283181fa..804cdae3 100644 --- a/secure/samples/health-risk-python/Dockerfile +++ b/secure/samples/health-risk-python/Dockerfile @@ -11,6 +11,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --locked --no-install-project COPY main.py pyproject.toml uv.lock ./ +COPY sce ./sce RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --locked diff --git a/secure/samples/health-risk-python/README.md b/secure/samples/health-risk-python/README.md index b5ae4977..3e7fa889 100644 --- a/secure/samples/health-risk-python/README.md +++ b/secure/samples/health-risk-python/README.md @@ -1,20 +1,19 @@ # health-risk-python -Sample Data Controller app (Python, pydantic). Reads the records file at -`RECORDS_PATH`, aggregates every record for that algorithm, and writes a single -JSON summary to `RESULT_PATH`. Progress lines go to stderr so the result holds -exactly one JSON object. +Sample Data Controller app (Python, pydantic). Calls sce-unit over the gRPC socket at +`SCE_RPC_SOCKET`, streams the algorithm's records with `FetchRecords`, aggregates +them, and submits a single result message with `SubmitResult`. Progress lines go +to stderr. ```json {"algorithmId": "662fa378-dc53-425d-9225-a187b11ea7dc", "userCount": 250, "avgRestingHeartRate": 66.79, "avgDailySteps": 7620.37, "avgSleepHours": 7.26, "riskFlaggedCount": 28, "riskFlaggedPct": 11.2, "byActivityLevel": {"sedentary": 52, "moderate": 138, "active": 60}} ``` -Build and run (same `{"algorithmId", "records"}` file sce-unit writes): +Build: ```sh docker build --platform linux/arm64 -t health-risk-python . -docker run --rm -v /tmp/sce-run:/data \ - -e RECORDS_PATH=/data/records.json \ - -e RESULT_PATH=/data/result.json \ - health-risk-python ``` + +Inside the enclave nsjail sets `SCE_RPC_SOCKET=/data/sce.sock` and `ALGORITHM_ID`; +sce-unit serves the gRPC API on that socket. diff --git a/secure/samples/health-risk-python/main.py b/secure/samples/health-risk-python/main.py index a527e67f..555261ac 100644 --- a/secure/samples/health-risk-python/main.py +++ b/secure/samples/health-risk-python/main.py @@ -1,9 +1,12 @@ -import json import os import sys +import grpc +from google.protobuf import json_format, struct_pb2 from pydantic import BaseModel +from sce.v1 import records_pb2, records_pb2_grpc + class Day(BaseModel): restingHeartRate: float @@ -81,39 +84,27 @@ def analyze(algorithm_id: str | None, records: list[Record]) -> Analysis: def required_env(name: str) -> str: value = os.environ.get(name) if not value: - raise ValueError(f"missing required env var {name}; run with RECORDS_PATH set") + raise ValueError(f"missing required env var {name}") return value -def load_records(path: str) -> tuple[str | None, list[Record]]: - try: - with open(path, "r", encoding="utf-8") as handle: - body = json.load(handle) - except (OSError, json.JSONDecodeError) as error: - raise ValueError(f"could not read records file {path}: {error}") from error - records = body["records"] if isinstance(body, dict) else body - algorithm_id = body.get("algorithmId") if isinstance(body, dict) else None - return algorithm_id, [Record.model_validate(record) for record in records] - - -def write_result(result: Analysis) -> None: - payload = result.model_dump_json() - path = os.environ.get("RESULT_PATH") - if path: - try: - with open(path, "w", encoding="utf-8") as handle: - handle.write(payload) - except OSError as error: - raise ValueError(f"could not write result to {path}: {error}") from error - else: - print(payload) - - def main() -> None: - records_path = required_env("RECORDS_PATH") - algorithm_id, records = load_records(records_path) - log(f"loaded {len(records)} records from {records_path}") - write_result(analyze(algorithm_id, records)) + algorithm_id = required_env("ALGORITHM_ID") + socket_path = required_env("SCE_RPC_SOCKET") + channel = grpc.insecure_channel(f"unix://{socket_path}") + try: + stub = records_pb2_grpc.DataAccessStub(channel) + records = [ + Record.model_validate({"data": json_format.MessageToDict(record.data)}) + for record in stub.FetchRecords(records_pb2.FetchRecordsRequest()) + ] + log(f"loaded {len(records)} records over {socket_path}") + result = analyze(algorithm_id, records) + payload = struct_pb2.Struct() + json_format.ParseDict(result.model_dump(), payload) + stub.SubmitResult(records_pb2.SubmitResultRequest(result=payload)) + finally: + channel.close() if __name__ == "__main__": diff --git a/secure/samples/health-risk-python/pyproject.toml b/secure/samples/health-risk-python/pyproject.toml index 85ab81be..5046055c 100644 --- a/secure/samples/health-risk-python/pyproject.toml +++ b/secure/samples/health-risk-python/pyproject.toml @@ -3,5 +3,7 @@ name = "health-risk-python" version = "0.1.0" requires-python = ">=3.12" dependencies = [ + "grpcio>=1.85", + "protobuf>=7.35,<8", "pydantic>=2.9", ] diff --git a/secure/samples/health-risk-python/sce/__init__.py b/secure/samples/health-risk-python/sce/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/secure/samples/health-risk-python/sce/v1/__init__.py b/secure/samples/health-risk-python/sce/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/secure/samples/health-risk-python/sce/v1/envelope_pb2.py b/secure/samples/health-risk-python/sce/v1/envelope_pb2.py new file mode 100644 index 00000000..6ddb22af --- /dev/null +++ b/secure/samples/health-risk-python/sce/v1/envelope_pb2.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: sce/v1/envelope.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 7, + 35, + 1, + '', + 'sce/v1/envelope.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15sce/v1/envelope.proto\x12\x06sce.v1\"D\n\x0eRecordEnvelope\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\r\n\x05nonce\x18\x02 \x01(\x0c\x12\x12\n\nciphertext\x18\x03 \x01(\x0c\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sce.v1.envelope_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_RECORDENVELOPE']._serialized_start=33 + _globals['_RECORDENVELOPE']._serialized_end=101 +# @@protoc_insertion_point(module_scope) diff --git a/secure/samples/health-risk-python/sce/v1/envelope_pb2_grpc.py b/secure/samples/health-risk-python/sce/v1/envelope_pb2_grpc.py new file mode 100644 index 00000000..ea3fb741 --- /dev/null +++ b/secure/samples/health-risk-python/sce/v1/envelope_pb2_grpc.py @@ -0,0 +1,24 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + + +GRPC_GENERATED_VERSION = '1.85.3' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in sce/v1/envelope_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) diff --git a/secure/samples/health-risk-python/sce/v1/records_pb2.py b/secure/samples/health-risk-python/sce/v1/records_pb2.py new file mode 100644 index 00000000..70148579 --- /dev/null +++ b/secure/samples/health-risk-python/sce/v1/records_pb2.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: sce/v1/records.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 7, + 35, + 1, + '', + 'sce/v1/records.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14sce/v1/records.proto\x12\x06sce.v1\x1a\x1cgoogle/protobuf/struct.proto\"B\n\x06Record\x12\x11\n\trecord_id\x18\x01 \x01(\t\x12%\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"\x15\n\x13\x46\x65tchRecordsRequest\">\n\x13SubmitResultRequest\x12\'\n\x06result\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"\x16\n\x14SubmitResultResponse2\x96\x01\n\nDataAccess\x12=\n\x0c\x46\x65tchRecords\x12\x1b.sce.v1.FetchRecordsRequest\x1a\x0e.sce.v1.Record0\x01\x12I\n\x0cSubmitResult\x12\x1b.sce.v1.SubmitResultRequest\x1a\x1c.sce.v1.SubmitResultResponseb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sce.v1.records_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_RECORD']._serialized_start=62 + _globals['_RECORD']._serialized_end=128 + _globals['_FETCHRECORDSREQUEST']._serialized_start=130 + _globals['_FETCHRECORDSREQUEST']._serialized_end=151 + _globals['_SUBMITRESULTREQUEST']._serialized_start=153 + _globals['_SUBMITRESULTREQUEST']._serialized_end=215 + _globals['_SUBMITRESULTRESPONSE']._serialized_start=217 + _globals['_SUBMITRESULTRESPONSE']._serialized_end=239 + _globals['_DATAACCESS']._serialized_start=242 + _globals['_DATAACCESS']._serialized_end=392 +# @@protoc_insertion_point(module_scope) diff --git a/secure/samples/health-risk-python/sce/v1/records_pb2_grpc.py b/secure/samples/health-risk-python/sce/v1/records_pb2_grpc.py new file mode 100644 index 00000000..34844934 --- /dev/null +++ b/secure/samples/health-risk-python/sce/v1/records_pb2_grpc.py @@ -0,0 +1,145 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +from sce.v1 import records_pb2 as sce_dot_v1_dot_records__pb2 + +GRPC_GENERATED_VERSION = '1.85.3' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in sce/v1/records_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class DataAccessStub: + """DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.FetchRecords = channel.unary_stream( + '/sce.v1.DataAccess/FetchRecords', + request_serializer=sce_dot_v1_dot_records__pb2.FetchRecordsRequest.SerializeToString, + response_deserializer=sce_dot_v1_dot_records__pb2.Record.FromString, + _registered_method=True) + self.SubmitResult = channel.unary_unary( + '/sce.v1.DataAccess/SubmitResult', + request_serializer=sce_dot_v1_dot_records__pb2.SubmitResultRequest.SerializeToString, + response_deserializer=sce_dot_v1_dot_records__pb2.SubmitResultResponse.FromString, + _registered_method=True) + + +class DataAccessServicer: + """DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. + """ + + def FetchRecords(self, request, context): + """Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SubmitResult(self, request, context): + """The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_DataAccessServicer_to_server(servicer, server): + rpc_method_handlers = { + 'FetchRecords': grpc.unary_stream_rpc_method_handler( + servicer.FetchRecords, + request_deserializer=sce_dot_v1_dot_records__pb2.FetchRecordsRequest.FromString, + response_serializer=sce_dot_v1_dot_records__pb2.Record.SerializeToString, + ), + 'SubmitResult': grpc.unary_unary_rpc_method_handler( + servicer.SubmitResult, + request_deserializer=sce_dot_v1_dot_records__pb2.SubmitResultRequest.FromString, + response_serializer=sce_dot_v1_dot_records__pb2.SubmitResultResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'sce.v1.DataAccess', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('sce.v1.DataAccess', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class DataAccess: + """DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. + """ + + @staticmethod + def FetchRecords(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/sce.v1.DataAccess/FetchRecords', + sce_dot_v1_dot_records__pb2.FetchRecordsRequest.SerializeToString, + sce_dot_v1_dot_records__pb2.Record.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def SubmitResult(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/sce.v1.DataAccess/SubmitResult', + sce_dot_v1_dot_records__pb2.SubmitResultRequest.SerializeToString, + sce_dot_v1_dot_records__pb2.SubmitResultResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/secure/samples/health-risk-python/uv.lock b/secure/samples/health-risk-python/uv.lock index e166be08..97b813ac 100644 --- a/secure/samples/health-risk-python/uv.lock +++ b/secure/samples/health-risk-python/uv.lock @@ -11,16 +11,88 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] +[[package]] +name = "grpcio" +version = "1.85.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/48/8e14d92cca434b448c7018654da80f574aea3663f7d1f87ed044455c45c6/grpcio-1.85.3.tar.gz", hash = "sha256:8e18f15d1b09b8771edbf59d634b26cbe4903bc2577cb667264934a6655eb8b4", size = 14103282, upload-time = "2026-11-18T04:28:00.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/60/cd483e718da0871333b2788eb9788a7cac82b8e8b0c4e741c3e17b6f9c06/grpcio-1.85.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:f87a4f69f1a8da71abe14c246442f0f39a6cf65638f1fafe099ca387d4e3b84a", size = 6796867, upload-time = "2026-11-18T04:26:38.316Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/bff4fb89a2db9b76ab42d0bc22d1006893037eb1d0f66e59fbb918418e34/grpcio-1.85.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:a53ea9200c626a9d3e3249175f9df4f1349ce863c0510d4b88ed653d976b19ff", size = 12633356, upload-time = "2026-11-18T04:26:40.309Z" }, + { url = "https://files.pythonhosted.org/packages/cc/5a/ae4c57adacc44900038f6ba68f2676916894badfa37dbcad3eede85bff50/grpcio-1.85.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:258c579112434464d24dd5f8f73bf4b220a9c88219bef4d6f56410b5363a0a81", size = 7170526, upload-time = "2026-11-18T04:26:42.632Z" }, + { url = "https://files.pythonhosted.org/packages/1b/42/8be0c18afc3ff9a83e5c949a894c291663fea1e37ab52049cf6d935d2017/grpcio-1.85.3-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:7ee86a10ec206fb0d0874fab4ba16723476aff4c087b3dc8910f628f103dc2d7", size = 8180283, upload-time = "2026-11-18T04:26:44.175Z" }, + { url = "https://files.pythonhosted.org/packages/68/4e/3748a2dc5be6c6d5099ec32f08123f67ddc13f9bcea2f3375abdddc1cb77/grpcio-1.85.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8266434d54c3442ebdba71d97e6f688b73dd00194bda6e1e35e57272afbd2d73", size = 7360708, upload-time = "2026-11-18T04:26:45.81Z" }, + { url = "https://files.pythonhosted.org/packages/e7/04/4a7147c5d331621e3fd043e61af40efbf1e7159d6007de9c0809ac4fc29b/grpcio-1.85.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:df376205244bf5639da75cd475ab52756c53573f18090fbcecb2211bc31201ab", size = 7928335, upload-time = "2026-11-18T04:26:47.371Z" }, + { url = "https://files.pythonhosted.org/packages/13/33/1dc6d16e3046410ee5a9a8b7c0045925103f5cbfaab09687d7d3cab2d43b/grpcio-1.85.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2aa5c328cc6409e1f40b1fcc7f3879ee7deb4a292821a6821dc8511eafe3a205", size = 9178894, upload-time = "2026-11-18T04:26:48.918Z" }, + { url = "https://files.pythonhosted.org/packages/6d/72/d8f9a450686d3671d2909a104248609529c743d0b59936cfc3a45e5a55bb/grpcio-1.85.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f13766ed972cda01690510f71e5b1a14132be83215129e800d950e917fe12edc", size = 8318194, upload-time = "2026-11-18T04:26:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/2e/cc/e5ac058d051ff964330219d1b8f32f0a96feaaba0e13114a0de6665287bf/grpcio-1.85.3-cp312-cp312-win32.whl", hash = "sha256:776a7f4ef1e437f38d37a535d18c30c01049aa40531100f203eb2b0e761fd0da", size = 4771559, upload-time = "2026-11-18T04:26:52.353Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9b/46844fe2ce358a63c21685510fc50e8b91d670ebeb510faeb7b19d1505d6/grpcio-1.85.3-cp312-cp312-win_amd64.whl", hash = "sha256:a0d02a0d5d267fcc9e6d141a7ffc609b09240607f890044881991ea9e660f7c7", size = 5607592, upload-time = "2026-11-18T04:26:53.924Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ad/31fa425090e5215201248321992ea0ba492a24150209a6c9e083b211585c/grpcio-1.85.3-cp313-cp313-linux_armv7l.whl", hash = "sha256:87d162186a98e907cdf2c3ee4f419967c513f9799fdf9168f5aeb7e7889f020e", size = 6801398, upload-time = "2026-11-18T04:26:55.726Z" }, + { url = "https://files.pythonhosted.org/packages/b1/a2/4fbd17b1222e42fccc74b9f3c55e93b09283f74906b6328ebc8356649407/grpcio-1.85.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:16903fb91a0e79c3cb3bf2e2f615782e03366d967b1f7e2419be11dcfb04380a", size = 12629676, upload-time = "2026-11-18T04:26:57.674Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f7/6c6ca236171d5b4bc8ab15cfa562679d4544407e1cb6f81f192f1dbd91b4/grpcio-1.85.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:399fc184b4d2dd0b014b4b880a5b1ecc99383b610dbb270acb43d1b23e1e8b9c", size = 7174532, upload-time = "2026-11-18T04:26:59.672Z" }, + { url = "https://files.pythonhosted.org/packages/2d/66/e7cf3d79d97a5cf4764a2ec2bb1eb652ae16e82edd64255f35d06baa7ee8/grpcio-1.85.3-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1fe3c07494cce4fa62e0732d3569f779e6ae04d70f73ef013fb2eb8f97382d80", size = 8185312, upload-time = "2026-11-18T04:27:01.484Z" }, + { url = "https://files.pythonhosted.org/packages/f8/0c/fd8fc7eb7d1ec942c11716105bca8314028409ca7955fd2aaf203e7fb8ef/grpcio-1.85.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d20015c0dfc91de7a9a037cc84c7a174e8fe62e6427f563f1e2e35ca10f3798", size = 7362466, upload-time = "2026-11-18T04:27:03.171Z" }, + { url = "https://files.pythonhosted.org/packages/ae/99/2e087c79e717321eb09bcbff9f352f61a487ff2740974e2b6fca3ca56462/grpcio-1.85.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:726a43169704bbc79e8aac0eeb6170e02db2fc9e3f4b76f4ecd2d265477e15ef", size = 7931006, upload-time = "2026-11-18T04:27:05.057Z" }, + { url = "https://files.pythonhosted.org/packages/db/2b/45b8d0646f7c7c5cf025e64ee066dba09c783b7669aae44ff4ab2754058d/grpcio-1.85.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f7adb3642a622fb31a1f03d39c5586bc73c849bb38b6e14fd45061ad66e6fa85", size = 9171504, upload-time = "2026-11-18T04:27:06.941Z" }, + { url = "https://files.pythonhosted.org/packages/46/40/f7dec903ffdc035abfa95a537f36d156b3dd32373ddc0b7027f633abf784/grpcio-1.85.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e5d3582c1124665b661c2e8a51dc17ba64f35b0b85d0b173e0e1361b398a5322", size = 8309787, upload-time = "2026-11-18T04:27:08.995Z" }, + { url = "https://files.pythonhosted.org/packages/ec/2e/9a4681c7683f383b49ca9fde12eb46cd5283f86fa0701c07ddf3ba21b1e2/grpcio-1.85.3-cp313-cp313-win32.whl", hash = "sha256:b02c67ceb00596cf1eb33457d1a81295d144e0584b92f7bfb22991b8c4351efd", size = 4768570, upload-time = "2026-11-18T04:27:10.774Z" }, + { url = "https://files.pythonhosted.org/packages/fd/9d/ca74ad4b5e0433f829aa570b872f9fe88f9937bb38ba633ab8c180e3b8ca/grpcio-1.85.3-cp313-cp313-win_amd64.whl", hash = "sha256:0998f5af9091eb333e6c731b84a8838d92489c367b3f1346d57e0490f981d8e1", size = 5607144, upload-time = "2026-11-18T04:27:12.479Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0e/a9058914124978273c2173c17e71945fb17096d2fa580614313efa1e4db1/grpcio-1.85.3-cp314-cp314-linux_armv7l.whl", hash = "sha256:21864eaf90c9f62f8dfb1a1ff52c9c3b303ae84c8c00173136dff4d049e69ed7", size = 6802798, upload-time = "2026-11-18T04:27:14.238Z" }, + { url = "https://files.pythonhosted.org/packages/e6/28/48f2f94fe423df1760c024483605af2421e48efa691016980c98d6456b21/grpcio-1.85.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:77567ff6786d26cd89c71b6d9da5b75398917e78f68e9d41487abcf213dcb6eb", size = 12640137, upload-time = "2026-11-18T04:27:16.208Z" }, + { url = "https://files.pythonhosted.org/packages/44/ce/29c23817115979837adaedfb5e17a5a4d2ad4995712f126fdda17a29e78c/grpcio-1.85.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8147e611f308598a3457e8f77dac9e201a80997bf7189b18e9b7799dab1e75d3", size = 7187528, upload-time = "2026-11-18T04:27:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/ca/90/373296f8925a329c5b7343e6052a64cea238acf90a1b74de30a9a01c0cc3/grpcio-1.85.3-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5ca5277d9cc880f792118612c5cef03f0521c1bbcbcc4bb2a3d1e6807cc146cd", size = 8187103, upload-time = "2026-11-18T04:27:19.821Z" }, + { url = "https://files.pythonhosted.org/packages/ad/0d/48be6838774fabe6237bd3c256da635d3cd5d194975eb42bc5963f8b65bc/grpcio-1.85.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d1c3193d7db9dc7786325b6f93c800b5eecb1037daf9b3ca105ef8ffa6607895", size = 7366699, upload-time = "2026-11-18T04:27:21.671Z" }, + { url = "https://files.pythonhosted.org/packages/49/19/5b961f1b9e2fd0fe8de82af542962e2493b502034e25b23f90602144890a/grpcio-1.85.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c0de93a8197bc51aabc930bae79e4d5e83259ffeb5f6901e5306d55e96feb7c5", size = 7940070, upload-time = "2026-11-18T04:27:31.744Z" }, + { url = "https://files.pythonhosted.org/packages/33/4f/bd3cc386afce28d1986f56ea340171490710861152afa0884ba4319e3297/grpcio-1.85.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f88c854546f3b95adf259226abecfc653c1e8355543cedd3fbaae1fc634ef072", size = 9174926, upload-time = "2026-11-18T04:27:33.554Z" }, + { url = "https://files.pythonhosted.org/packages/2a/11/61c8bce50f533bb889da9c4e2959aa9beca94afd6feb69eee0a26ba24571/grpcio-1.85.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:24a29604524c608a9b79fa556849f3dab7c07b0ec0acf57d84aa298c5f7fb18b", size = 8315381, upload-time = "2026-11-18T04:27:35.624Z" }, + { url = "https://files.pythonhosted.org/packages/c1/27/5ed2a3b1e88c73c4fbcd18e8512e0d24e10e0f13f6c075a6c8dfe96683bb/grpcio-1.85.3-cp314-cp314-win32.whl", hash = "sha256:ddbe20f752db7cd96dc2a33dfb2a4a45584b259cdd6c44d8a681d831317d5583", size = 4854323, upload-time = "2026-11-18T04:27:37.73Z" }, + { url = "https://files.pythonhosted.org/packages/78/13/53a87a76f6fecc346a053dc9aa861a89467da2aae9af44ded1d26bf3ce38/grpcio-1.85.3-cp314-cp314-win_amd64.whl", hash = "sha256:31cf4daac3d9bd4c4a8d195f05cc46ae0242aa16622186be8cb7c3f7481089de", size = 5758444, upload-time = "2026-11-18T04:27:39.197Z" }, + { url = "https://files.pythonhosted.org/packages/ee/c7/e79469ca6b361caf0e3b1ea035a5cd76bafe5fabca00f7ea7de6a9cf4174/grpcio-1.85.3-cp315-cp315-linux_armv7l.whl", hash = "sha256:5ba281614699dd37ae420664f462bc17ce7dcf192d9d02236afc8152a97da08e", size = 6803083, upload-time = "2026-11-18T04:27:40.78Z" }, + { url = "https://files.pythonhosted.org/packages/f7/64/4282c2ab2aa09c3095a0775104f035ae3e938a5d06cf0f83ca8b6f2f72fe/grpcio-1.85.3-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:e739cc3edb632553b7d3a1b7ff344025a3adb033e8eb4a193fe1fb386322d186", size = 12657854, upload-time = "2026-11-18T04:27:43.126Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f8/08e1d186003d416f800a8dda961be3de9f73981bac6c2ded472920670a88/grpcio-1.85.3-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:af380cfb3c46cb21582999784cdfbce89adde3b269505f0b3bb8da297e6cd929", size = 7209473, upload-time = "2026-11-18T04:27:45.583Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6f/a46b468fac256ee6f301ad078c4e8738e2d492795fe298179a39e04afdcb/grpcio-1.85.3-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:53443ce822ace2723a27a1d94d7ae4c9a49fbab888ff9bad9c501b8d6cdc6339", size = 7510210, upload-time = "2026-11-18T04:27:47.489Z" }, + { url = "https://files.pythonhosted.org/packages/d8/2a/6a6641ecd03615d840bd819feddd46e76eb1fccf47d91fb015a8255a3046/grpcio-1.85.3-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b80182b312759d0eebd7aa4dc13ad2ba2de0441398dbc25a5d1aa8e0c0101070", size = 7409583, upload-time = "2026-11-18T04:27:49.306Z" }, + { url = "https://files.pythonhosted.org/packages/98/30/742e190fd71466b72cb502789fa3afdfd077cce7c87f2fcbab644d5d8a9b/grpcio-1.85.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:f047dc3ab821621c86f5c99cd376414681d8e00dfa6a98d01ba68fbabbeae1f5", size = 7962909, upload-time = "2026-11-18T04:27:51.366Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b7/0542a74f844f8b9ea59b969ed19563dd0f3ed0dc512aade29360fe45b363/grpcio-1.85.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:42324e16026c9d6c817718e5a471cc37bd99401bba6a8078243b0f6ab03d13a1", size = 8501815, upload-time = "2026-11-18T04:27:53.621Z" }, + { url = "https://files.pythonhosted.org/packages/49/a1/3fcd7edce1e828d7bee4a830f7f7982884b26ca0da140473443977f9f2b1/grpcio-1.85.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:0aa95286ed10b23c71f3179bad4a52c7ba92a6624c306511876d530f92c951a0", size = 8345782, upload-time = "2026-11-18T04:27:55.432Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a4/6ef6fc206a3ab51f30186d882034216e4fa3af6aebc10fbeb2d8a7caab4b/grpcio-1.85.3-cp315-cp315-win32.whl", hash = "sha256:df6de234f7ee8b65177ef731decc88495dbed42446c4cf007a411280ffbda86c", size = 4854418, upload-time = "2026-11-18T04:27:56.957Z" }, + { url = "https://files.pythonhosted.org/packages/2e/59/e55fc8322982b0dd54dabfc7660d56e5dd1bf6c5ecfbe84c556b4d826365/grpcio-1.85.3-cp315-cp315-win_amd64.whl", hash = "sha256:db5a410f61d4030a1d605e482095c216e3ff107571d37f2de5005c23d336b7ef", size = 5758673, upload-time = "2026-11-18T04:27:58.474Z" }, +] + [[package]] name = "health-risk-python" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, { name = "pydantic" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = ">=2.9" }] +requires-dist = [ + { name = "grpcio", specifier = ">=1.85" }, + { name = "protobuf", specifier = ">=7.35,<8" }, + { name = "pydantic", specifier = ">=2.9" }, +] + +[[package]] +name = "protobuf" +version = "7.36.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/89/5b8517baa72f84a67b8a307ba953c91057af618bf40bf676f3c03551f8f0/protobuf-7.36.2.tar.gz", hash = "sha256:497d0463ff3316681da6c0b9e8d06cb465d61abce00b613ab42226175644d1bb", size = 512737, upload-time = "2026-09-17T20:07:59.326Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/72/98342feb672507c8f3a69e34b4fa8961f608edba5c1a48a6f47156d92cb5/protobuf-7.36.2-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:cbc70b17ee27e28894c7fee8bb04be1abead49e936bc70eb60052531eee2079e", size = 456039, upload-time = "2026-09-17T20:07:51.542Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ea/91fdf7c2b8bbd49cde056f00a9df6773532987e1c00fe2830b895af95c7e/protobuf-7.36.2-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:e11e1f0180583a2af89db6a2ecd9e8dc40aa6d2988ca175bfd0e6d12ea72d74e", size = 344219, upload-time = "2026-09-17T20:07:52.914Z" }, + { url = "https://files.pythonhosted.org/packages/17/ab/5fd5f8ece73fad885c5a09aa849b32d70472f954ba3a92d3bb5974ea953b/protobuf-7.36.2-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:f4fee11ec330d238b34a05c9b675f693c20415d1c5bd7d5320cc2f8a798eb9cf", size = 357223, upload-time = "2026-09-17T20:07:53.985Z" }, + { url = "https://files.pythonhosted.org/packages/db/f3/3996583dd2906297a637af12114deddf7658af6e683fedb83be061983fb5/protobuf-7.36.2-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2", size = 343223, upload-time = "2026-09-17T20:07:54.931Z" }, + { url = "https://files.pythonhosted.org/packages/fc/1b/dcc64f358fcb51811b58ae40b3d28f820725f116d86487cc20bd4b130701/protobuf-7.36.2-cp310-abi3-win32.whl", hash = "sha256:912c1221170e16c08d1f086762f563dd61ff83c18b5fa6652952dfaded66f728", size = 442998, upload-time = "2026-09-17T20:07:55.826Z" }, + { url = "https://files.pythonhosted.org/packages/8a/55/b77bda4e5e5f5971fb51b07663694690e9afdb9402136c16a522bd621cad/protobuf-7.36.2-cp310-abi3-win_amd64.whl", hash = "sha256:a300819d441e078a5608c0d3c709796bb548136058fda017ae51d425b44fd353", size = 456514, upload-time = "2026-09-17T20:07:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/d52c7016b04b6c5108f26691f9d33ec82a9b65d041f1a9c771137693d618/protobuf-7.36.2-py3-none-any.whl", hash = "sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e", size = 179806, upload-time = "2026-09-17T20:07:58.211Z" }, +] [[package]] name = "pydantic" diff --git a/secure/samples/product-trends-node/Dockerfile b/secure/samples/product-trends-node/Dockerfile index 8121e7ba..53a70d80 100644 --- a/secure/samples/product-trends-node/Dockerfile +++ b/secure/samples/product-trends-node/Dockerfile @@ -1,5 +1,10 @@ -FROM node:20-alpine +FROM node:22-alpine -COPY main.js package.json /app/ +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev + +COPY main.js ./ +COPY gen ./gen ENTRYPOINT ["node", "/app/main.js"] diff --git a/secure/samples/product-trends-node/README.md b/secure/samples/product-trends-node/README.md index b8109fff..c093fe00 100644 --- a/secure/samples/product-trends-node/README.md +++ b/secure/samples/product-trends-node/README.md @@ -1,20 +1,19 @@ # product-trends-node -Sample Data Controller app (Node `node:fs`, no deps). Reads the records file at -`RECORDS_PATH`, aggregates every record for that algorithm, and writes a single -JSON summary to `RESULT_PATH`. Progress lines go to stderr so the result holds -exactly one JSON object. +Sample Data Controller app (Node, @grpc/grpc-js). Calls sce-unit over the gRPC socket at +`SCE_RPC_SOCKET`, streams the algorithm's records with `FetchRecords`, aggregates +them, and submits a single result message with `SubmitResult`. Progress lines go +to stderr. ```json {"algorithmId": "3b8bba4a-8106-4309-9446-7bce99a1453a", "userCount": 2500, "totalRevenueCents": 2004647586, "totalOrders": 26264, "topCategoriesByAgeBand": {}} ``` -Build and run (same `{"algorithmId", "records"}` file sce-unit writes): +Build: ```sh docker build --platform linux/arm64 -t product-trends-node . -docker run --rm -v /tmp/sce-run:/data \ - -e RECORDS_PATH=/data/records.json \ - -e RESULT_PATH=/data/result.json \ - product-trends-node ``` + +Inside the enclave nsjail sets `SCE_RPC_SOCKET=/data/sce.sock` and `ALGORITHM_ID`; +sce-unit serves the gRPC API on that socket. diff --git a/secure/samples/product-trends-node/gen/sce/v1/envelope_pb.d.ts b/secure/samples/product-trends-node/gen/sce/v1/envelope_pb.d.ts new file mode 100644 index 00000000..fe81e431 --- /dev/null +++ b/secure/samples/product-trends-node/gen/sce/v1/envelope_pb.d.ts @@ -0,0 +1,41 @@ +// @generated by protoc-gen-es v2.17.0 with parameter "target=js+dts" +// @generated from file sce/v1/envelope.proto (package sce.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; + +/** + * Describes the file sce/v1/envelope.proto. + */ +export declare const file_sce_v1_envelope: GenFile; + +/** + * RecordEnvelope is what pdm-service stores in DynamoDB Data. ciphertext is AES-256-GCM over a serialized + * sce.v1.Record; nonce is the 96-bit GCM nonce, and the auth tag is appended to ciphertext (Go GCM layout). + * + * @generated from message sce.v1.RecordEnvelope + */ +export declare type RecordEnvelope = Message<"sce.v1.RecordEnvelope"> & { + /** + * @generated from field: uint32 version = 1; + */ + version: number; + + /** + * @generated from field: bytes nonce = 2; + */ + nonce: Uint8Array; + + /** + * @generated from field: bytes ciphertext = 3; + */ + ciphertext: Uint8Array; +}; + +/** + * Describes the message sce.v1.RecordEnvelope. + * Use `create(RecordEnvelopeSchema)` to create a new message. + */ +export declare const RecordEnvelopeSchema: GenMessage; + diff --git a/secure/samples/product-trends-node/gen/sce/v1/envelope_pb.js b/secure/samples/product-trends-node/gen/sce/v1/envelope_pb.js new file mode 100644 index 00000000..23234e77 --- /dev/null +++ b/secure/samples/product-trends-node/gen/sce/v1/envelope_pb.js @@ -0,0 +1,19 @@ +// @generated by protoc-gen-es v2.17.0 with parameter "target=js+dts" +// @generated from file sce/v1/envelope.proto (package sce.v1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; + +/** + * Describes the file sce/v1/envelope.proto. + */ +export const file_sce_v1_envelope = /*@__PURE__*/ + fileDesc("ChVzY2UvdjEvZW52ZWxvcGUucHJvdG8SBnNjZS52MSJECg5SZWNvcmRFbnZlbG9wZRIPCgd2ZXJzaW9uGAEgASgNEg0KBW5vbmNlGAIgASgMEhIKCmNpcGhlcnRleHQYAyABKAxiBnByb3RvMw"); + +/** + * Describes the message sce.v1.RecordEnvelope. + * Use `create(RecordEnvelopeSchema)` to create a new message. + */ +export const RecordEnvelopeSchema = /*@__PURE__*/ + messageDesc(file_sce_v1_envelope, 0); + diff --git a/secure/samples/product-trends-node/gen/sce/v1/records_pb.d.ts b/secure/samples/product-trends-node/gen/sce/v1/records_pb.d.ts new file mode 100644 index 00000000..db715791 --- /dev/null +++ b/secure/samples/product-trends-node/gen/sce/v1/records_pb.d.ts @@ -0,0 +1,104 @@ +// @generated by protoc-gen-es v2.17.0 with parameter "target=js+dts" +// @generated from file sce/v1/records.proto (package sce.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import type { JsonObject, Message } from "@bufbuild/protobuf"; + +/** + * Describes the file sce/v1/records.proto. + */ +export declare const file_sce_v1_records: GenFile; + +/** + * Record is one personal-data record handed to the DC application. record_id stays separate so the DC app + * never sees data subject ids; data is schemaless per dataset, so it travels as a Struct. + * + * @generated from message sce.v1.Record + */ +export declare type Record = Message<"sce.v1.Record"> & { + /** + * @generated from field: string record_id = 1; + */ + recordId: string; + + /** + * @generated from field: google.protobuf.Struct data = 2; + */ + data?: JsonObject | undefined; +}; + +/** + * Describes the message sce.v1.Record. + * Use `create(RecordSchema)` to create a new message. + */ +export declare const RecordSchema: GenMessage; + +/** + * @generated from message sce.v1.FetchRecordsRequest + */ +export declare type FetchRecordsRequest = Message<"sce.v1.FetchRecordsRequest"> & { +}; + +/** + * Describes the message sce.v1.FetchRecordsRequest. + * Use `create(FetchRecordsRequestSchema)` to create a new message. + */ +export declare const FetchRecordsRequestSchema: GenMessage; + +/** + * @generated from message sce.v1.SubmitResultRequest + */ +export declare type SubmitResultRequest = Message<"sce.v1.SubmitResultRequest"> & { + /** + * @generated from field: google.protobuf.Struct result = 1; + */ + result?: JsonObject | undefined; +}; + +/** + * Describes the message sce.v1.SubmitResultRequest. + * Use `create(SubmitResultRequestSchema)` to create a new message. + */ +export declare const SubmitResultRequestSchema: GenMessage; + +/** + * @generated from message sce.v1.SubmitResultResponse + */ +export declare type SubmitResultResponse = Message<"sce.v1.SubmitResultResponse"> & { +}; + +/** + * Describes the message sce.v1.SubmitResultResponse. + * Use `create(SubmitResultResponseSchema)` to create a new message. + */ +export declare const SubmitResultResponseSchema: GenMessage; + +/** + * DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. + * + * @generated from service sce.v1.DataAccess + */ +export declare const DataAccess: GenService<{ + /** + * Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. + * + * @generated from rpc sce.v1.DataAccess.FetchRecords + */ + fetchRecords: { + methodKind: "server_streaming"; + input: typeof FetchRecordsRequestSchema; + output: typeof RecordSchema; + }, + /** + * The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. + * + * @generated from rpc sce.v1.DataAccess.SubmitResult + */ + submitResult: { + methodKind: "unary"; + input: typeof SubmitResultRequestSchema; + output: typeof SubmitResultResponseSchema; + }, +}>; + diff --git a/secure/samples/product-trends-node/gen/sce/v1/records_pb.js b/secure/samples/product-trends-node/gen/sce/v1/records_pb.js new file mode 100644 index 00000000..90b3b809 --- /dev/null +++ b/secure/samples/product-trends-node/gen/sce/v1/records_pb.js @@ -0,0 +1,49 @@ +// @generated by protoc-gen-es v2.17.0 with parameter "target=js+dts" +// @generated from file sce/v1/records.proto (package sce.v1, syntax proto3) +/* eslint-disable */ + +import { fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv2"; +import { file_google_protobuf_struct } from "@bufbuild/protobuf/wkt"; + +/** + * Describes the file sce/v1/records.proto. + */ +export const file_sce_v1_records = /*@__PURE__*/ + fileDesc("ChRzY2UvdjEvcmVjb3Jkcy5wcm90bxIGc2NlLnYxIkIKBlJlY29yZBIRCglyZWNvcmRfaWQYASABKAkSJQoEZGF0YRgCIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QiFQoTRmV0Y2hSZWNvcmRzUmVxdWVzdCI+ChNTdWJtaXRSZXN1bHRSZXF1ZXN0EicKBnJlc3VsdBgBIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QiFgoUU3VibWl0UmVzdWx0UmVzcG9uc2UylgEKCkRhdGFBY2Nlc3MSPQoMRmV0Y2hSZWNvcmRzEhsuc2NlLnYxLkZldGNoUmVjb3Jkc1JlcXVlc3QaDi5zY2UudjEuUmVjb3JkMAESSQoMU3VibWl0UmVzdWx0Ehsuc2NlLnYxLlN1Ym1pdFJlc3VsdFJlcXVlc3QaHC5zY2UudjEuU3VibWl0UmVzdWx0UmVzcG9uc2ViBnByb3RvMw", [file_google_protobuf_struct]); + +/** + * Describes the message sce.v1.Record. + * Use `create(RecordSchema)` to create a new message. + */ +export const RecordSchema = /*@__PURE__*/ + messageDesc(file_sce_v1_records, 0); + +/** + * Describes the message sce.v1.FetchRecordsRequest. + * Use `create(FetchRecordsRequestSchema)` to create a new message. + */ +export const FetchRecordsRequestSchema = /*@__PURE__*/ + messageDesc(file_sce_v1_records, 1); + +/** + * Describes the message sce.v1.SubmitResultRequest. + * Use `create(SubmitResultRequestSchema)` to create a new message. + */ +export const SubmitResultRequestSchema = /*@__PURE__*/ + messageDesc(file_sce_v1_records, 2); + +/** + * Describes the message sce.v1.SubmitResultResponse. + * Use `create(SubmitResultResponseSchema)` to create a new message. + */ +export const SubmitResultResponseSchema = /*@__PURE__*/ + messageDesc(file_sce_v1_records, 3); + +/** + * DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. + * + * @generated from service sce.v1.DataAccess + */ +export const DataAccess = /*@__PURE__*/ + serviceDesc(file_sce_v1_records, 0); + diff --git a/secure/samples/product-trends-node/main.js b/secure/samples/product-trends-node/main.js index f2f6a933..931addba 100644 --- a/secure/samples/product-trends-node/main.js +++ b/secure/samples/product-trends-node/main.js @@ -1,6 +1,16 @@ -const fs = require("node:fs"); +import * as grpc from "@grpc/grpc-js"; +import { create, fromBinary, fromJson, toBinary, toJson } from "@bufbuild/protobuf"; +import { StructSchema } from "@bufbuild/protobuf/wkt"; +import { + FetchRecordsRequestSchema, + RecordSchema, + SubmitResultRequestSchema, + SubmitResultResponseSchema, +} from "./gen/sce/v1/records_pb.js"; const AGE_BANDS = ["18-29", "30-44", "45-59", "60+"]; +const FETCH_RECORDS = "/sce.v1.DataAccess/FetchRecords"; +const SUBMIT_RESULT = "/sce.v1.DataAccess/SubmitResult"; function ageBand(age) { if (age < 30) return "18-29"; @@ -44,26 +54,54 @@ function analyze(algorithmId, records) { }; } -function loadRecords(path) { - const body = JSON.parse(fs.readFileSync(path, "utf8")); - return { algorithmId: body.algorithmId, records: body.records }; +function fetchRecords(client) { + return new Promise((resolve, reject) => { + const records = []; + const call = client.makeServerStreamRequest( + FETCH_RECORDS, + (message) => toBinary(FetchRecordsRequestSchema, message), + (bytes) => fromBinary(RecordSchema, bytes), + create(FetchRecordsRequestSchema), + ); + call.on("data", (record) => records.push(record)); + call.on("error", reject); + call.on("end", () => resolve(records)); + }); } -function writeResult(result) { - const payload = JSON.stringify(result); - if (process.env.RESULT_PATH) { - fs.writeFileSync(process.env.RESULT_PATH, payload); - } else { - console.log(payload); - } +function submitResult(client, result) { + return new Promise((resolve, reject) => { + const request = create(SubmitResultRequestSchema, { result: fromJson(StructSchema, result) }); + client.makeUnaryRequest( + SUBMIT_RESULT, + (message) => toBinary(SubmitResultRequestSchema, message), + (bytes) => fromBinary(SubmitResultResponseSchema, bytes), + request, + (error) => (error ? reject(error) : resolve()), + ); + }); } -function main() { - const recordsPath = process.env.RECORDS_PATH; - if (!recordsPath) throw new Error("RECORDS_PATH is required"); - const { algorithmId, records } = loadRecords(recordsPath); - console.error(`loaded ${records.length} records from ${recordsPath}`); - writeResult(analyze(algorithmId, records)); +async function main() { + const socketPath = process.env.SCE_RPC_SOCKET; + if (!socketPath) throw new Error("SCE_RPC_SOCKET is required"); + const algorithmId = process.env.ALGORITHM_ID; + if (!algorithmId) throw new Error("ALGORITHM_ID is required"); + + const client = new grpc.Client(`unix:${socketPath}`, grpc.credentials.createInsecure()); + try { + const records = (await fetchRecords(client)).map((record) => ({ + recordId: record.recordId, + data: record.data ? toJson(StructSchema, record.data) : {}, + })); + console.error(`loaded ${records.length} records over ${socketPath}`); + await submitResult(client, analyze(algorithmId, records)); + } finally { + client.close(); + } } -main(); +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/secure/samples/product-trends-node/package-lock.json b/secure/samples/product-trends-node/package-lock.json new file mode 100644 index 00000000..611d2224 --- /dev/null +++ b/secure/samples/product-trends-node/package-lock.json @@ -0,0 +1,347 @@ +{ + "name": "product-trends-node", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "product-trends-node", + "version": "0.1.0", + "dependencies": { + "@bufbuild/protobuf": "^2.10.0", + "@grpc/grpc-js": "^1.14.0" + } + }, + "node_modules/@bufbuild/protobuf": { + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.17.0.tgz", + "integrity": "sha512-hcPkSz6D7MiTU/5VBBhK+kxaYDoOdQ6yJRFJ3kS8Nydp+VZXfg1vrpABKPNod8bXMufsVCmiUJVIKFzCzta4uw==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.7", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.7.tgz", + "integrity": "sha512-W4W/fcBMGRVtG6dW2c7cw2mnpS3hvbBd+oQXjh2iYFTZDk1JJmcNqkM5KSaPubQpamlUUurrlb7G1s9NJeERmQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/node": { + "version": "26.6.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.6.tgz", + "integrity": "sha512-MzQj+8Kx3XI40OmUAE6Qoh58ZMNstNNMwn2nfd3qWsfe7nDNSqJNzes4njhz6AHwrSHOffSLwXU6yb474Q4aog==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + } + } +} diff --git a/secure/samples/product-trends-node/package.json b/secure/samples/product-trends-node/package.json index 779da8ad..25bee4cf 100644 --- a/secure/samples/product-trends-node/package.json +++ b/secure/samples/product-trends-node/package.json @@ -5,5 +5,9 @@ "type": "module", "scripts": { "start": "node main.js" + }, + "dependencies": { + "@bufbuild/protobuf": "^2.10.0", + "@grpc/grpc-js": "^1.14.0" } } diff --git a/secure/samples/spend-analysis-go/Dockerfile b/secure/samples/spend-analysis-go/Dockerfile index 3dccd8a1..91b428b6 100644 --- a/secure/samples/spend-analysis-go/Dockerfile +++ b/secure/samples/spend-analysis-go/Dockerfile @@ -1,6 +1,7 @@ -FROM golang:1.24-alpine AS build +FROM golang:1.26-alpine AS build WORKDIR /src -COPY go.mod main.go ./ +COPY go.mod go.sum main.go ./ +COPY gen ./gen RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /spend-analysis . FROM scratch diff --git a/secure/samples/spend-analysis-go/README.md b/secure/samples/spend-analysis-go/README.md index 9c525f22..e9a25696 100644 --- a/secure/samples/spend-analysis-go/README.md +++ b/secure/samples/spend-analysis-go/README.md @@ -1,20 +1,19 @@ # spend-analysis-go -Sample Data Controller app (Go stdlib only). Reads the records file at -`RECORDS_PATH`, aggregates every record for that algorithm, and writes a single -JSON summary to `RESULT_PATH`. Progress lines go to stderr so the result holds -exactly one JSON object. +Sample Data Controller app (Go, grpc-go). Calls sce-unit over the gRPC socket at +`SCE_RPC_SOCKET`, streams the algorithm's records with `FetchRecords`, aggregates +them, and submits a single result message with `SubmitResult`. Progress lines go +to stderr. ```json {"algorithmId": "c7cc14fa-e962-4548-aff4-a54148798cd8", "userCount": 1000, "totalSpendByCategory": {"dining": 697211.38}, "avgMonthlySpendPerUser": 1199.77, "anomalies": [], "anomalyUserCount": 969} ``` -Build and run (same `{"algorithmId", "records"}` file sce-unit writes): +Build: ```sh docker build --platform linux/arm64 -t spend-analysis-go . -docker run --rm -v /tmp/sce-run:/data \ - -e RECORDS_PATH=/data/records.json \ - -e RESULT_PATH=/data/result.json \ - spend-analysis-go ``` + +Inside the enclave nsjail sets `SCE_RPC_SOCKET=/data/sce.sock` and `ALGORITHM_ID`; +sce-unit serves the gRPC API on that socket. diff --git a/secure/samples/spend-analysis-go/gen/scev1/envelope.pb.go b/secure/samples/spend-analysis-go/gen/scev1/envelope.pb.go new file mode 100644 index 00000000..06627d8f --- /dev/null +++ b/secure/samples/spend-analysis-go/gen/scev1/envelope.pb.go @@ -0,0 +1,144 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.6 +// protoc v7.35.1 +// source: sce/v1/envelope.proto + +package scev1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// RecordEnvelope is what pdm-service stores in DynamoDB Data. ciphertext is AES-256-GCM over a serialized +// sce.v1.Record; nonce is the 96-bit GCM nonce, and the auth tag is appended to ciphertext (Go GCM layout). +type RecordEnvelope struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version uint32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + Nonce []byte `protobuf:"bytes,2,opt,name=nonce,proto3" json:"nonce,omitempty"` + Ciphertext []byte `protobuf:"bytes,3,opt,name=ciphertext,proto3" json:"ciphertext,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RecordEnvelope) Reset() { + *x = RecordEnvelope{} + mi := &file_sce_v1_envelope_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RecordEnvelope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RecordEnvelope) ProtoMessage() {} + +func (x *RecordEnvelope) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_envelope_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RecordEnvelope.ProtoReflect.Descriptor instead. +func (*RecordEnvelope) Descriptor() ([]byte, []int) { + return file_sce_v1_envelope_proto_rawDescGZIP(), []int{0} +} + +func (x *RecordEnvelope) GetVersion() uint32 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *RecordEnvelope) GetNonce() []byte { + if x != nil { + return x.Nonce + } + return nil +} + +func (x *RecordEnvelope) GetCiphertext() []byte { + if x != nil { + return x.Ciphertext + } + return nil +} + +var File_sce_v1_envelope_proto protoreflect.FileDescriptor + +const file_sce_v1_envelope_proto_rawDesc = "" + + "\n" + + "\x15sce/v1/envelope.proto\x12\x06sce.v1\"`\n" + + "\x0eRecordEnvelope\x12\x18\n" + + "\aversion\x18\x01 \x01(\rR\aversion\x12\x14\n" + + "\x05nonce\x18\x02 \x01(\fR\x05nonce\x12\x1e\n" + + "\n" + + "ciphertext\x18\x03 \x01(\fR\n" + + "ciphertextb\x06proto3" + +var ( + file_sce_v1_envelope_proto_rawDescOnce sync.Once + file_sce_v1_envelope_proto_rawDescData []byte +) + +func file_sce_v1_envelope_proto_rawDescGZIP() []byte { + file_sce_v1_envelope_proto_rawDescOnce.Do(func() { + file_sce_v1_envelope_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sce_v1_envelope_proto_rawDesc), len(file_sce_v1_envelope_proto_rawDesc))) + }) + return file_sce_v1_envelope_proto_rawDescData +} + +var file_sce_v1_envelope_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_sce_v1_envelope_proto_goTypes = []any{ + (*RecordEnvelope)(nil), // 0: sce.v1.RecordEnvelope +} +var file_sce_v1_envelope_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_sce_v1_envelope_proto_init() } +func file_sce_v1_envelope_proto_init() { + if File_sce_v1_envelope_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_sce_v1_envelope_proto_rawDesc), len(file_sce_v1_envelope_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_sce_v1_envelope_proto_goTypes, + DependencyIndexes: file_sce_v1_envelope_proto_depIdxs, + MessageInfos: file_sce_v1_envelope_proto_msgTypes, + }.Build() + File_sce_v1_envelope_proto = out.File + file_sce_v1_envelope_proto_goTypes = nil + file_sce_v1_envelope_proto_depIdxs = nil +} diff --git a/secure/samples/spend-analysis-go/gen/scev1/records.pb.go b/secure/samples/spend-analysis-go/gen/scev1/records.pb.go new file mode 100644 index 00000000..b4a17f7a --- /dev/null +++ b/secure/samples/spend-analysis-go/gen/scev1/records.pb.go @@ -0,0 +1,268 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.6 +// protoc v7.35.1 +// source: sce/v1/records.proto + +package scev1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + structpb "google.golang.org/protobuf/types/known/structpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Record is one personal-data record handed to the DC application. record_id stays separate so the DC app +// never sees data subject ids; data is schemaless per dataset, so it travels as a Struct. +type Record struct { + state protoimpl.MessageState `protogen:"open.v1"` + RecordId string `protobuf:"bytes,1,opt,name=record_id,json=recordId,proto3" json:"record_id,omitempty"` + Data *structpb.Struct `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Record) Reset() { + *x = Record{} + mi := &file_sce_v1_records_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Record) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Record) ProtoMessage() {} + +func (x *Record) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_records_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Record.ProtoReflect.Descriptor instead. +func (*Record) Descriptor() ([]byte, []int) { + return file_sce_v1_records_proto_rawDescGZIP(), []int{0} +} + +func (x *Record) GetRecordId() string { + if x != nil { + return x.RecordId + } + return "" +} + +func (x *Record) GetData() *structpb.Struct { + if x != nil { + return x.Data + } + return nil +} + +type FetchRecordsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FetchRecordsRequest) Reset() { + *x = FetchRecordsRequest{} + mi := &file_sce_v1_records_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FetchRecordsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchRecordsRequest) ProtoMessage() {} + +func (x *FetchRecordsRequest) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_records_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchRecordsRequest.ProtoReflect.Descriptor instead. +func (*FetchRecordsRequest) Descriptor() ([]byte, []int) { + return file_sce_v1_records_proto_rawDescGZIP(), []int{1} +} + +type SubmitResultRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Result *structpb.Struct `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubmitResultRequest) Reset() { + *x = SubmitResultRequest{} + mi := &file_sce_v1_records_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubmitResultRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubmitResultRequest) ProtoMessage() {} + +func (x *SubmitResultRequest) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_records_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubmitResultRequest.ProtoReflect.Descriptor instead. +func (*SubmitResultRequest) Descriptor() ([]byte, []int) { + return file_sce_v1_records_proto_rawDescGZIP(), []int{2} +} + +func (x *SubmitResultRequest) GetResult() *structpb.Struct { + if x != nil { + return x.Result + } + return nil +} + +type SubmitResultResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubmitResultResponse) Reset() { + *x = SubmitResultResponse{} + mi := &file_sce_v1_records_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubmitResultResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubmitResultResponse) ProtoMessage() {} + +func (x *SubmitResultResponse) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_records_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubmitResultResponse.ProtoReflect.Descriptor instead. +func (*SubmitResultResponse) Descriptor() ([]byte, []int) { + return file_sce_v1_records_proto_rawDescGZIP(), []int{3} +} + +var File_sce_v1_records_proto protoreflect.FileDescriptor + +const file_sce_v1_records_proto_rawDesc = "" + + "\n" + + "\x14sce/v1/records.proto\x12\x06sce.v1\x1a\x1cgoogle/protobuf/struct.proto\"R\n" + + "\x06Record\x12\x1b\n" + + "\trecord_id\x18\x01 \x01(\tR\brecordId\x12+\n" + + "\x04data\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x04data\"\x15\n" + + "\x13FetchRecordsRequest\"F\n" + + "\x13SubmitResultRequest\x12/\n" + + "\x06result\x18\x01 \x01(\v2\x17.google.protobuf.StructR\x06result\"\x16\n" + + "\x14SubmitResultResponse2\x96\x01\n" + + "\n" + + "DataAccess\x12=\n" + + "\fFetchRecords\x12\x1b.sce.v1.FetchRecordsRequest\x1a\x0e.sce.v1.Record0\x01\x12I\n" + + "\fSubmitResult\x12\x1b.sce.v1.SubmitResultRequest\x1a\x1c.sce.v1.SubmitResultResponseb\x06proto3" + +var ( + file_sce_v1_records_proto_rawDescOnce sync.Once + file_sce_v1_records_proto_rawDescData []byte +) + +func file_sce_v1_records_proto_rawDescGZIP() []byte { + file_sce_v1_records_proto_rawDescOnce.Do(func() { + file_sce_v1_records_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sce_v1_records_proto_rawDesc), len(file_sce_v1_records_proto_rawDesc))) + }) + return file_sce_v1_records_proto_rawDescData +} + +var file_sce_v1_records_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_sce_v1_records_proto_goTypes = []any{ + (*Record)(nil), // 0: sce.v1.Record + (*FetchRecordsRequest)(nil), // 1: sce.v1.FetchRecordsRequest + (*SubmitResultRequest)(nil), // 2: sce.v1.SubmitResultRequest + (*SubmitResultResponse)(nil), // 3: sce.v1.SubmitResultResponse + (*structpb.Struct)(nil), // 4: google.protobuf.Struct +} +var file_sce_v1_records_proto_depIdxs = []int32{ + 4, // 0: sce.v1.Record.data:type_name -> google.protobuf.Struct + 4, // 1: sce.v1.SubmitResultRequest.result:type_name -> google.protobuf.Struct + 1, // 2: sce.v1.DataAccess.FetchRecords:input_type -> sce.v1.FetchRecordsRequest + 2, // 3: sce.v1.DataAccess.SubmitResult:input_type -> sce.v1.SubmitResultRequest + 0, // 4: sce.v1.DataAccess.FetchRecords:output_type -> sce.v1.Record + 3, // 5: sce.v1.DataAccess.SubmitResult:output_type -> sce.v1.SubmitResultResponse + 4, // [4:6] is the sub-list for method output_type + 2, // [2:4] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_sce_v1_records_proto_init() } +func file_sce_v1_records_proto_init() { + if File_sce_v1_records_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_sce_v1_records_proto_rawDesc), len(file_sce_v1_records_proto_rawDesc)), + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_sce_v1_records_proto_goTypes, + DependencyIndexes: file_sce_v1_records_proto_depIdxs, + MessageInfos: file_sce_v1_records_proto_msgTypes, + }.Build() + File_sce_v1_records_proto = out.File + file_sce_v1_records_proto_goTypes = nil + file_sce_v1_records_proto_depIdxs = nil +} diff --git a/secure/samples/spend-analysis-go/gen/scev1/records_grpc.pb.go b/secure/samples/spend-analysis-go/gen/scev1/records_grpc.pb.go new file mode 100644 index 00000000..2e22911a --- /dev/null +++ b/secure/samples/spend-analysis-go/gen/scev1/records_grpc.pb.go @@ -0,0 +1,171 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v7.35.1 +// source: sce/v1/records.proto + +package scev1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + DataAccess_FetchRecords_FullMethodName = "/sce.v1.DataAccess/FetchRecords" + DataAccess_SubmitResult_FullMethodName = "/sce.v1.DataAccess/SubmitResult" +) + +// DataAccessClient is the client API for DataAccess service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. +type DataAccessClient interface { + // Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. + FetchRecords(ctx context.Context, in *FetchRecordsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Record], error) + // The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. + SubmitResult(ctx context.Context, in *SubmitResultRequest, opts ...grpc.CallOption) (*SubmitResultResponse, error) +} + +type dataAccessClient struct { + cc grpc.ClientConnInterface +} + +func NewDataAccessClient(cc grpc.ClientConnInterface) DataAccessClient { + return &dataAccessClient{cc} +} + +func (c *dataAccessClient) FetchRecords(ctx context.Context, in *FetchRecordsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Record], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &DataAccess_ServiceDesc.Streams[0], DataAccess_FetchRecords_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[FetchRecordsRequest, Record]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type DataAccess_FetchRecordsClient = grpc.ServerStreamingClient[Record] + +func (c *dataAccessClient) SubmitResult(ctx context.Context, in *SubmitResultRequest, opts ...grpc.CallOption) (*SubmitResultResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SubmitResultResponse) + err := c.cc.Invoke(ctx, DataAccess_SubmitResult_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// DataAccessServer is the server API for DataAccess service. +// All implementations must embed UnimplementedDataAccessServer +// for forward compatibility. +// +// DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. +type DataAccessServer interface { + // Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. + FetchRecords(*FetchRecordsRequest, grpc.ServerStreamingServer[Record]) error + // The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. + SubmitResult(context.Context, *SubmitResultRequest) (*SubmitResultResponse, error) + mustEmbedUnimplementedDataAccessServer() +} + +// UnimplementedDataAccessServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedDataAccessServer struct{} + +func (UnimplementedDataAccessServer) FetchRecords(*FetchRecordsRequest, grpc.ServerStreamingServer[Record]) error { + return status.Errorf(codes.Unimplemented, "method FetchRecords not implemented") +} +func (UnimplementedDataAccessServer) SubmitResult(context.Context, *SubmitResultRequest) (*SubmitResultResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SubmitResult not implemented") +} +func (UnimplementedDataAccessServer) mustEmbedUnimplementedDataAccessServer() {} +func (UnimplementedDataAccessServer) testEmbeddedByValue() {} + +// UnsafeDataAccessServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to DataAccessServer will +// result in compilation errors. +type UnsafeDataAccessServer interface { + mustEmbedUnimplementedDataAccessServer() +} + +func RegisterDataAccessServer(s grpc.ServiceRegistrar, srv DataAccessServer) { + // If the following call pancis, it indicates UnimplementedDataAccessServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&DataAccess_ServiceDesc, srv) +} + +func _DataAccess_FetchRecords_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(FetchRecordsRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(DataAccessServer).FetchRecords(m, &grpc.GenericServerStream[FetchRecordsRequest, Record]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type DataAccess_FetchRecordsServer = grpc.ServerStreamingServer[Record] + +func _DataAccess_SubmitResult_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SubmitResultRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DataAccessServer).SubmitResult(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DataAccess_SubmitResult_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DataAccessServer).SubmitResult(ctx, req.(*SubmitResultRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// DataAccess_ServiceDesc is the grpc.ServiceDesc for DataAccess service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var DataAccess_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "sce.v1.DataAccess", + HandlerType: (*DataAccessServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SubmitResult", + Handler: _DataAccess_SubmitResult_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "FetchRecords", + Handler: _DataAccess_FetchRecords_Handler, + ServerStreams: true, + }, + }, + Metadata: "sce/v1/records.proto", +} diff --git a/secure/samples/spend-analysis-go/go.mod b/secure/samples/spend-analysis-go/go.mod index 5462b7e3..fee4f251 100644 --- a/secure/samples/spend-analysis-go/go.mod +++ b/secure/samples/spend-analysis-go/go.mod @@ -1,3 +1,15 @@ module spend-analysis -go 1.24 +go 1.26.0 + +require ( + google.golang.org/grpc v1.85.0 + google.golang.org/protobuf v1.36.12 +) + +require ( + golang.org/x/net v0.60.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect +) diff --git a/secure/samples/spend-analysis-go/go.sum b/secure/samples/spend-analysis-go/go.sum new file mode 100644 index 00000000..cae8887c --- /dev/null +++ b/secure/samples/spend-analysis-go/go.sum @@ -0,0 +1,18 @@ +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/secure/samples/spend-analysis-go/main.go b/secure/samples/spend-analysis-go/main.go index 6b515dc3..416f3e06 100644 --- a/secure/samples/spend-analysis-go/main.go +++ b/secure/samples/spend-analysis-go/main.go @@ -1,12 +1,22 @@ package main import ( + "context" "encoding/json" "fmt" + "io" "log" "math" + "net" "os" "sort" + + scev1 "spend-analysis/gen/scev1" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/structpb" ) type Transaction struct { @@ -27,11 +37,6 @@ type Record struct { Data Data `json:"data"` } -type RecordsFile struct { - AlgorithmID string `json:"algorithmId"` - Records []Record `json:"records"` -} - type Anomaly struct { RecordID string `json:"recordId"` Count int `json:"count"` @@ -107,44 +112,80 @@ func analyze(algorithmID string, records []Record) Result { } } -func loadRecords(path string) (string, []Record, error) { - body, err := os.ReadFile(path) +func fetchRecords(ctx context.Context, client scev1.DataAccessClient) ([]Record, error) { + stream, err := client.FetchRecords(ctx, &scev1.FetchRecordsRequest{}) if err != nil { - return "", nil, fmt.Errorf("read %s: %w", path, err) + return nil, fmt.Errorf("fetch records: %w", err) } - var file RecordsFile - if err := json.Unmarshal(body, &file); err != nil { - return "", nil, fmt.Errorf("parse %s: %w", path, err) + + var records []Record + for { + message, err := stream.Recv() + if err == io.EOF { + return records, nil + } + if err != nil { + return nil, fmt.Errorf("receive record: %w", err) + } + raw, err := protojson.Marshal(message.GetData()) + if err != nil { + return nil, fmt.Errorf("encode record %s: %w", message.GetRecordId(), err) + } + var data Data + if err := json.Unmarshal(raw, &data); err != nil { + return nil, fmt.Errorf("decode record %s: %w", message.GetRecordId(), err) + } + records = append(records, Record{RecordID: message.GetRecordId(), Data: data}) } - return file.AlgorithmID, file.Records, nil } -func writeResult(result Result) error { - out, err := json.Marshal(result) +func submitResult(ctx context.Context, client scev1.DataAccessClient, result Result) error { + raw, err := json.Marshal(result) if err != nil { return fmt.Errorf("marshal result: %w", err) } - if path := os.Getenv("RESULT_PATH"); path != "" { - return os.WriteFile(path, out, 0o644) + payload := &structpb.Struct{} + if err := protojson.Unmarshal(raw, payload); err != nil { + return fmt.Errorf("encode result: %w", err) + } + if _, err := client.SubmitResult(ctx, &scev1.SubmitResultRequest{Result: payload}); err != nil { + return fmt.Errorf("submit result: %w", err) } - fmt.Println(string(out)) return nil } func main() { log.SetOutput(os.Stderr) - recordsPath := os.Getenv("RECORDS_PATH") - if recordsPath == "" { - log.Fatal("RECORDS_PATH is required") + socketPath := os.Getenv("SCE_RPC_SOCKET") + if socketPath == "" { + log.Fatal("SCE_RPC_SOCKET is required") + } + algorithmID := os.Getenv("ALGORITHM_ID") + if algorithmID == "" { + log.Fatal("ALGORITHM_ID is required") } - algorithmID, records, err := loadRecords(recordsPath) + conn, err := grpc.NewClient( + "passthrough:///sce-unit", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socketPath) + }), + ) + if err != nil { + log.Fatalf("dial %s: %v", socketPath, err) + } + defer conn.Close() + + ctx := context.Background() + client := scev1.NewDataAccessClient(conn) + records, err := fetchRecords(ctx, client) if err != nil { log.Fatalf("load: %v", err) } - log.Printf("loaded %d records from %s", len(records), recordsPath) + log.Printf("loaded %d records over %s", len(records), socketPath) - if err := writeResult(analyze(algorithmID, records)); err != nil { - log.Fatalf("write result: %v", err) + if err := submitResult(ctx, client, analyze(algorithmID, records)); err != nil { + log.Fatalf("result: %v", err) } -} \ No newline at end of file +} diff --git a/secure/samples/trial-eligibility-python/Dockerfile b/secure/samples/trial-eligibility-python/Dockerfile index 283181fa..804cdae3 100644 --- a/secure/samples/trial-eligibility-python/Dockerfile +++ b/secure/samples/trial-eligibility-python/Dockerfile @@ -11,6 +11,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --locked --no-install-project COPY main.py pyproject.toml uv.lock ./ +COPY sce ./sce RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --locked diff --git a/secure/samples/trial-eligibility-python/README.md b/secure/samples/trial-eligibility-python/README.md index 495f20ca..c3568912 100644 --- a/secure/samples/trial-eligibility-python/README.md +++ b/secure/samples/trial-eligibility-python/README.md @@ -1,20 +1,19 @@ # trial-eligibility-python -Sample Data Controller app (Python, pydantic). Reads the records file at -`RECORDS_PATH`, aggregates every record for that algorithm, and writes a single -JSON summary to `RESULT_PATH`. Progress lines go to stderr so the result holds -exactly one JSON object. +Sample Data Controller app (Python, pydantic). Calls sce-unit over the gRPC socket at +`SCE_RPC_SOCKET`, streams the algorithm's records with `FetchRecords`, aggregates +them, and submits a single result message with `SubmitResult`. Progress lines go +to stderr. ```json {"algorithmId": "e587067e-47dc-4835-8f87-5e66fc979adf", "userCount": 10000, "eligibleCount": 282, "eligiblePct": 2.82, "exclusionBreakdown": {"age": 5093, "hba1c": 8628, "egfr": 1590, "ckd": 2545, "bmi": 4378}, "eligibleRecordIds": ["user-000019", "user-000023", "..."]} ``` -Build and run (same `{"algorithmId", "records"}` file sce-unit writes): +Build: ```sh docker build --platform linux/arm64 -t trial-eligibility-python . -docker run --rm -v /tmp/sce-run:/data \ - -e RECORDS_PATH=/data/records.json \ - -e RESULT_PATH=/data/result.json \ - trial-eligibility-python ``` + +Inside the enclave nsjail sets `SCE_RPC_SOCKET=/data/sce.sock` and `ALGORITHM_ID`; +sce-unit serves the gRPC API on that socket. diff --git a/secure/samples/trial-eligibility-python/main.py b/secure/samples/trial-eligibility-python/main.py index 40f6b668..32f77d16 100644 --- a/secure/samples/trial-eligibility-python/main.py +++ b/secure/samples/trial-eligibility-python/main.py @@ -1,9 +1,12 @@ -import json import os import sys +import grpc +from google.protobuf import json_format, struct_pb2 from pydantic import BaseModel +from sce.v1 import records_pb2, records_pb2_grpc + class Labs(BaseModel): hba1c: float @@ -74,39 +77,30 @@ def analyze(algorithm_id: str | None, records: list[Record]) -> Analysis: def required_env(name: str) -> str: value = os.environ.get(name) if not value: - raise ValueError(f"missing required env var {name}; run with RECORDS_PATH set") + raise ValueError(f"missing required env var {name}") return value -def load_records(path: str) -> tuple[str | None, list[Record]]: - try: - with open(path, "r", encoding="utf-8") as handle: - body = json.load(handle) - except (OSError, json.JSONDecodeError) as error: - raise ValueError(f"could not read records file {path}: {error}") from error - records = body["records"] if isinstance(body, dict) else body - algorithm_id = body.get("algorithmId") if isinstance(body, dict) else None - return algorithm_id, [Record.model_validate(record) for record in records] - - -def write_result(result: Analysis) -> None: - payload = result.model_dump_json() - path = os.environ.get("RESULT_PATH") - if path: - try: - with open(path, "w", encoding="utf-8") as handle: - handle.write(payload) - except OSError as error: - raise ValueError(f"could not write result to {path}: {error}") from error - else: - print(payload) - - def main() -> None: - records_path = required_env("RECORDS_PATH") - algorithm_id, records = load_records(records_path) - log(f"loaded {len(records)} records from {records_path}") - write_result(analyze(algorithm_id, records)) + algorithm_id = required_env("ALGORITHM_ID") + socket_path = required_env("SCE_RPC_SOCKET") + channel = grpc.insecure_channel(f"unix://{socket_path}") + try: + stub = records_pb2_grpc.DataAccessStub(channel) + records = [ + Record( + recordId=record.record_id, + data=RecordData.model_validate(json_format.MessageToDict(record.data)), + ) + for record in stub.FetchRecords(records_pb2.FetchRecordsRequest()) + ] + log(f"loaded {len(records)} records over {socket_path}") + result = analyze(algorithm_id, records) + payload = struct_pb2.Struct() + json_format.ParseDict(result.model_dump(), payload) + stub.SubmitResult(records_pb2.SubmitResultRequest(result=payload)) + finally: + channel.close() if __name__ == "__main__": diff --git a/secure/samples/trial-eligibility-python/pyproject.toml b/secure/samples/trial-eligibility-python/pyproject.toml index ef23f585..d264be14 100644 --- a/secure/samples/trial-eligibility-python/pyproject.toml +++ b/secure/samples/trial-eligibility-python/pyproject.toml @@ -3,5 +3,7 @@ name = "trial-eligibility-python" version = "0.1.0" requires-python = ">=3.12" dependencies = [ + "grpcio>=1.85", + "protobuf>=7.35,<8", "pydantic>=2.9", ] diff --git a/secure/samples/trial-eligibility-python/sce/__init__.py b/secure/samples/trial-eligibility-python/sce/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/secure/samples/trial-eligibility-python/sce/v1/__init__.py b/secure/samples/trial-eligibility-python/sce/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/secure/samples/trial-eligibility-python/sce/v1/envelope_pb2.py b/secure/samples/trial-eligibility-python/sce/v1/envelope_pb2.py new file mode 100644 index 00000000..6ddb22af --- /dev/null +++ b/secure/samples/trial-eligibility-python/sce/v1/envelope_pb2.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: sce/v1/envelope.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 7, + 35, + 1, + '', + 'sce/v1/envelope.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15sce/v1/envelope.proto\x12\x06sce.v1\"D\n\x0eRecordEnvelope\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\r\n\x05nonce\x18\x02 \x01(\x0c\x12\x12\n\nciphertext\x18\x03 \x01(\x0c\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sce.v1.envelope_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_RECORDENVELOPE']._serialized_start=33 + _globals['_RECORDENVELOPE']._serialized_end=101 +# @@protoc_insertion_point(module_scope) diff --git a/secure/samples/trial-eligibility-python/sce/v1/envelope_pb2_grpc.py b/secure/samples/trial-eligibility-python/sce/v1/envelope_pb2_grpc.py new file mode 100644 index 00000000..ea3fb741 --- /dev/null +++ b/secure/samples/trial-eligibility-python/sce/v1/envelope_pb2_grpc.py @@ -0,0 +1,24 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + + +GRPC_GENERATED_VERSION = '1.85.3' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in sce/v1/envelope_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) diff --git a/secure/samples/trial-eligibility-python/sce/v1/records_pb2.py b/secure/samples/trial-eligibility-python/sce/v1/records_pb2.py new file mode 100644 index 00000000..70148579 --- /dev/null +++ b/secure/samples/trial-eligibility-python/sce/v1/records_pb2.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: sce/v1/records.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 7, + 35, + 1, + '', + 'sce/v1/records.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14sce/v1/records.proto\x12\x06sce.v1\x1a\x1cgoogle/protobuf/struct.proto\"B\n\x06Record\x12\x11\n\trecord_id\x18\x01 \x01(\t\x12%\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"\x15\n\x13\x46\x65tchRecordsRequest\">\n\x13SubmitResultRequest\x12\'\n\x06result\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"\x16\n\x14SubmitResultResponse2\x96\x01\n\nDataAccess\x12=\n\x0c\x46\x65tchRecords\x12\x1b.sce.v1.FetchRecordsRequest\x1a\x0e.sce.v1.Record0\x01\x12I\n\x0cSubmitResult\x12\x1b.sce.v1.SubmitResultRequest\x1a\x1c.sce.v1.SubmitResultResponseb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sce.v1.records_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_RECORD']._serialized_start=62 + _globals['_RECORD']._serialized_end=128 + _globals['_FETCHRECORDSREQUEST']._serialized_start=130 + _globals['_FETCHRECORDSREQUEST']._serialized_end=151 + _globals['_SUBMITRESULTREQUEST']._serialized_start=153 + _globals['_SUBMITRESULTREQUEST']._serialized_end=215 + _globals['_SUBMITRESULTRESPONSE']._serialized_start=217 + _globals['_SUBMITRESULTRESPONSE']._serialized_end=239 + _globals['_DATAACCESS']._serialized_start=242 + _globals['_DATAACCESS']._serialized_end=392 +# @@protoc_insertion_point(module_scope) diff --git a/secure/samples/trial-eligibility-python/sce/v1/records_pb2_grpc.py b/secure/samples/trial-eligibility-python/sce/v1/records_pb2_grpc.py new file mode 100644 index 00000000..34844934 --- /dev/null +++ b/secure/samples/trial-eligibility-python/sce/v1/records_pb2_grpc.py @@ -0,0 +1,145 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +from sce.v1 import records_pb2 as sce_dot_v1_dot_records__pb2 + +GRPC_GENERATED_VERSION = '1.85.3' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in sce/v1/records_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class DataAccessStub: + """DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.FetchRecords = channel.unary_stream( + '/sce.v1.DataAccess/FetchRecords', + request_serializer=sce_dot_v1_dot_records__pb2.FetchRecordsRequest.SerializeToString, + response_deserializer=sce_dot_v1_dot_records__pb2.Record.FromString, + _registered_method=True) + self.SubmitResult = channel.unary_unary( + '/sce.v1.DataAccess/SubmitResult', + request_serializer=sce_dot_v1_dot_records__pb2.SubmitResultRequest.SerializeToString, + response_deserializer=sce_dot_v1_dot_records__pb2.SubmitResultResponse.FromString, + _registered_method=True) + + +class DataAccessServicer: + """DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. + """ + + def FetchRecords(self, request, context): + """Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SubmitResult(self, request, context): + """The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_DataAccessServicer_to_server(servicer, server): + rpc_method_handlers = { + 'FetchRecords': grpc.unary_stream_rpc_method_handler( + servicer.FetchRecords, + request_deserializer=sce_dot_v1_dot_records__pb2.FetchRecordsRequest.FromString, + response_serializer=sce_dot_v1_dot_records__pb2.Record.SerializeToString, + ), + 'SubmitResult': grpc.unary_unary_rpc_method_handler( + servicer.SubmitResult, + request_deserializer=sce_dot_v1_dot_records__pb2.SubmitResultRequest.FromString, + response_serializer=sce_dot_v1_dot_records__pb2.SubmitResultResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'sce.v1.DataAccess', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('sce.v1.DataAccess', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class DataAccess: + """DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. + """ + + @staticmethod + def FetchRecords(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/sce.v1.DataAccess/FetchRecords', + sce_dot_v1_dot_records__pb2.FetchRecordsRequest.SerializeToString, + sce_dot_v1_dot_records__pb2.Record.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def SubmitResult(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/sce.v1.DataAccess/SubmitResult', + sce_dot_v1_dot_records__pb2.SubmitResultRequest.SerializeToString, + sce_dot_v1_dot_records__pb2.SubmitResultResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/secure/samples/trial-eligibility-python/uv.lock b/secure/samples/trial-eligibility-python/uv.lock index 49088dc4..3fb80659 100644 --- a/secure/samples/trial-eligibility-python/uv.lock +++ b/secure/samples/trial-eligibility-python/uv.lock @@ -11,6 +11,72 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] +[[package]] +name = "grpcio" +version = "1.85.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/48/8e14d92cca434b448c7018654da80f574aea3663f7d1f87ed044455c45c6/grpcio-1.85.3.tar.gz", hash = "sha256:8e18f15d1b09b8771edbf59d634b26cbe4903bc2577cb667264934a6655eb8b4", size = 14103282, upload-time = "2026-11-18T04:28:00.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/60/cd483e718da0871333b2788eb9788a7cac82b8e8b0c4e741c3e17b6f9c06/grpcio-1.85.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:f87a4f69f1a8da71abe14c246442f0f39a6cf65638f1fafe099ca387d4e3b84a", size = 6796867, upload-time = "2026-11-18T04:26:38.316Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/bff4fb89a2db9b76ab42d0bc22d1006893037eb1d0f66e59fbb918418e34/grpcio-1.85.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:a53ea9200c626a9d3e3249175f9df4f1349ce863c0510d4b88ed653d976b19ff", size = 12633356, upload-time = "2026-11-18T04:26:40.309Z" }, + { url = "https://files.pythonhosted.org/packages/cc/5a/ae4c57adacc44900038f6ba68f2676916894badfa37dbcad3eede85bff50/grpcio-1.85.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:258c579112434464d24dd5f8f73bf4b220a9c88219bef4d6f56410b5363a0a81", size = 7170526, upload-time = "2026-11-18T04:26:42.632Z" }, + { url = "https://files.pythonhosted.org/packages/1b/42/8be0c18afc3ff9a83e5c949a894c291663fea1e37ab52049cf6d935d2017/grpcio-1.85.3-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:7ee86a10ec206fb0d0874fab4ba16723476aff4c087b3dc8910f628f103dc2d7", size = 8180283, upload-time = "2026-11-18T04:26:44.175Z" }, + { url = "https://files.pythonhosted.org/packages/68/4e/3748a2dc5be6c6d5099ec32f08123f67ddc13f9bcea2f3375abdddc1cb77/grpcio-1.85.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8266434d54c3442ebdba71d97e6f688b73dd00194bda6e1e35e57272afbd2d73", size = 7360708, upload-time = "2026-11-18T04:26:45.81Z" }, + { url = "https://files.pythonhosted.org/packages/e7/04/4a7147c5d331621e3fd043e61af40efbf1e7159d6007de9c0809ac4fc29b/grpcio-1.85.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:df376205244bf5639da75cd475ab52756c53573f18090fbcecb2211bc31201ab", size = 7928335, upload-time = "2026-11-18T04:26:47.371Z" }, + { url = "https://files.pythonhosted.org/packages/13/33/1dc6d16e3046410ee5a9a8b7c0045925103f5cbfaab09687d7d3cab2d43b/grpcio-1.85.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2aa5c328cc6409e1f40b1fcc7f3879ee7deb4a292821a6821dc8511eafe3a205", size = 9178894, upload-time = "2026-11-18T04:26:48.918Z" }, + { url = "https://files.pythonhosted.org/packages/6d/72/d8f9a450686d3671d2909a104248609529c743d0b59936cfc3a45e5a55bb/grpcio-1.85.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f13766ed972cda01690510f71e5b1a14132be83215129e800d950e917fe12edc", size = 8318194, upload-time = "2026-11-18T04:26:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/2e/cc/e5ac058d051ff964330219d1b8f32f0a96feaaba0e13114a0de6665287bf/grpcio-1.85.3-cp312-cp312-win32.whl", hash = "sha256:776a7f4ef1e437f38d37a535d18c30c01049aa40531100f203eb2b0e761fd0da", size = 4771559, upload-time = "2026-11-18T04:26:52.353Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9b/46844fe2ce358a63c21685510fc50e8b91d670ebeb510faeb7b19d1505d6/grpcio-1.85.3-cp312-cp312-win_amd64.whl", hash = "sha256:a0d02a0d5d267fcc9e6d141a7ffc609b09240607f890044881991ea9e660f7c7", size = 5607592, upload-time = "2026-11-18T04:26:53.924Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ad/31fa425090e5215201248321992ea0ba492a24150209a6c9e083b211585c/grpcio-1.85.3-cp313-cp313-linux_armv7l.whl", hash = "sha256:87d162186a98e907cdf2c3ee4f419967c513f9799fdf9168f5aeb7e7889f020e", size = 6801398, upload-time = "2026-11-18T04:26:55.726Z" }, + { url = "https://files.pythonhosted.org/packages/b1/a2/4fbd17b1222e42fccc74b9f3c55e93b09283f74906b6328ebc8356649407/grpcio-1.85.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:16903fb91a0e79c3cb3bf2e2f615782e03366d967b1f7e2419be11dcfb04380a", size = 12629676, upload-time = "2026-11-18T04:26:57.674Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f7/6c6ca236171d5b4bc8ab15cfa562679d4544407e1cb6f81f192f1dbd91b4/grpcio-1.85.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:399fc184b4d2dd0b014b4b880a5b1ecc99383b610dbb270acb43d1b23e1e8b9c", size = 7174532, upload-time = "2026-11-18T04:26:59.672Z" }, + { url = "https://files.pythonhosted.org/packages/2d/66/e7cf3d79d97a5cf4764a2ec2bb1eb652ae16e82edd64255f35d06baa7ee8/grpcio-1.85.3-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1fe3c07494cce4fa62e0732d3569f779e6ae04d70f73ef013fb2eb8f97382d80", size = 8185312, upload-time = "2026-11-18T04:27:01.484Z" }, + { url = "https://files.pythonhosted.org/packages/f8/0c/fd8fc7eb7d1ec942c11716105bca8314028409ca7955fd2aaf203e7fb8ef/grpcio-1.85.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d20015c0dfc91de7a9a037cc84c7a174e8fe62e6427f563f1e2e35ca10f3798", size = 7362466, upload-time = "2026-11-18T04:27:03.171Z" }, + { url = "https://files.pythonhosted.org/packages/ae/99/2e087c79e717321eb09bcbff9f352f61a487ff2740974e2b6fca3ca56462/grpcio-1.85.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:726a43169704bbc79e8aac0eeb6170e02db2fc9e3f4b76f4ecd2d265477e15ef", size = 7931006, upload-time = "2026-11-18T04:27:05.057Z" }, + { url = "https://files.pythonhosted.org/packages/db/2b/45b8d0646f7c7c5cf025e64ee066dba09c783b7669aae44ff4ab2754058d/grpcio-1.85.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f7adb3642a622fb31a1f03d39c5586bc73c849bb38b6e14fd45061ad66e6fa85", size = 9171504, upload-time = "2026-11-18T04:27:06.941Z" }, + { url = "https://files.pythonhosted.org/packages/46/40/f7dec903ffdc035abfa95a537f36d156b3dd32373ddc0b7027f633abf784/grpcio-1.85.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e5d3582c1124665b661c2e8a51dc17ba64f35b0b85d0b173e0e1361b398a5322", size = 8309787, upload-time = "2026-11-18T04:27:08.995Z" }, + { url = "https://files.pythonhosted.org/packages/ec/2e/9a4681c7683f383b49ca9fde12eb46cd5283f86fa0701c07ddf3ba21b1e2/grpcio-1.85.3-cp313-cp313-win32.whl", hash = "sha256:b02c67ceb00596cf1eb33457d1a81295d144e0584b92f7bfb22991b8c4351efd", size = 4768570, upload-time = "2026-11-18T04:27:10.774Z" }, + { url = "https://files.pythonhosted.org/packages/fd/9d/ca74ad4b5e0433f829aa570b872f9fe88f9937bb38ba633ab8c180e3b8ca/grpcio-1.85.3-cp313-cp313-win_amd64.whl", hash = "sha256:0998f5af9091eb333e6c731b84a8838d92489c367b3f1346d57e0490f981d8e1", size = 5607144, upload-time = "2026-11-18T04:27:12.479Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0e/a9058914124978273c2173c17e71945fb17096d2fa580614313efa1e4db1/grpcio-1.85.3-cp314-cp314-linux_armv7l.whl", hash = "sha256:21864eaf90c9f62f8dfb1a1ff52c9c3b303ae84c8c00173136dff4d049e69ed7", size = 6802798, upload-time = "2026-11-18T04:27:14.238Z" }, + { url = "https://files.pythonhosted.org/packages/e6/28/48f2f94fe423df1760c024483605af2421e48efa691016980c98d6456b21/grpcio-1.85.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:77567ff6786d26cd89c71b6d9da5b75398917e78f68e9d41487abcf213dcb6eb", size = 12640137, upload-time = "2026-11-18T04:27:16.208Z" }, + { url = "https://files.pythonhosted.org/packages/44/ce/29c23817115979837adaedfb5e17a5a4d2ad4995712f126fdda17a29e78c/grpcio-1.85.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8147e611f308598a3457e8f77dac9e201a80997bf7189b18e9b7799dab1e75d3", size = 7187528, upload-time = "2026-11-18T04:27:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/ca/90/373296f8925a329c5b7343e6052a64cea238acf90a1b74de30a9a01c0cc3/grpcio-1.85.3-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5ca5277d9cc880f792118612c5cef03f0521c1bbcbcc4bb2a3d1e6807cc146cd", size = 8187103, upload-time = "2026-11-18T04:27:19.821Z" }, + { url = "https://files.pythonhosted.org/packages/ad/0d/48be6838774fabe6237bd3c256da635d3cd5d194975eb42bc5963f8b65bc/grpcio-1.85.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d1c3193d7db9dc7786325b6f93c800b5eecb1037daf9b3ca105ef8ffa6607895", size = 7366699, upload-time = "2026-11-18T04:27:21.671Z" }, + { url = "https://files.pythonhosted.org/packages/49/19/5b961f1b9e2fd0fe8de82af542962e2493b502034e25b23f90602144890a/grpcio-1.85.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c0de93a8197bc51aabc930bae79e4d5e83259ffeb5f6901e5306d55e96feb7c5", size = 7940070, upload-time = "2026-11-18T04:27:31.744Z" }, + { url = "https://files.pythonhosted.org/packages/33/4f/bd3cc386afce28d1986f56ea340171490710861152afa0884ba4319e3297/grpcio-1.85.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f88c854546f3b95adf259226abecfc653c1e8355543cedd3fbaae1fc634ef072", size = 9174926, upload-time = "2026-11-18T04:27:33.554Z" }, + { url = "https://files.pythonhosted.org/packages/2a/11/61c8bce50f533bb889da9c4e2959aa9beca94afd6feb69eee0a26ba24571/grpcio-1.85.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:24a29604524c608a9b79fa556849f3dab7c07b0ec0acf57d84aa298c5f7fb18b", size = 8315381, upload-time = "2026-11-18T04:27:35.624Z" }, + { url = "https://files.pythonhosted.org/packages/c1/27/5ed2a3b1e88c73c4fbcd18e8512e0d24e10e0f13f6c075a6c8dfe96683bb/grpcio-1.85.3-cp314-cp314-win32.whl", hash = "sha256:ddbe20f752db7cd96dc2a33dfb2a4a45584b259cdd6c44d8a681d831317d5583", size = 4854323, upload-time = "2026-11-18T04:27:37.73Z" }, + { url = "https://files.pythonhosted.org/packages/78/13/53a87a76f6fecc346a053dc9aa861a89467da2aae9af44ded1d26bf3ce38/grpcio-1.85.3-cp314-cp314-win_amd64.whl", hash = "sha256:31cf4daac3d9bd4c4a8d195f05cc46ae0242aa16622186be8cb7c3f7481089de", size = 5758444, upload-time = "2026-11-18T04:27:39.197Z" }, + { url = "https://files.pythonhosted.org/packages/ee/c7/e79469ca6b361caf0e3b1ea035a5cd76bafe5fabca00f7ea7de6a9cf4174/grpcio-1.85.3-cp315-cp315-linux_armv7l.whl", hash = "sha256:5ba281614699dd37ae420664f462bc17ce7dcf192d9d02236afc8152a97da08e", size = 6803083, upload-time = "2026-11-18T04:27:40.78Z" }, + { url = "https://files.pythonhosted.org/packages/f7/64/4282c2ab2aa09c3095a0775104f035ae3e938a5d06cf0f83ca8b6f2f72fe/grpcio-1.85.3-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:e739cc3edb632553b7d3a1b7ff344025a3adb033e8eb4a193fe1fb386322d186", size = 12657854, upload-time = "2026-11-18T04:27:43.126Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f8/08e1d186003d416f800a8dda961be3de9f73981bac6c2ded472920670a88/grpcio-1.85.3-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:af380cfb3c46cb21582999784cdfbce89adde3b269505f0b3bb8da297e6cd929", size = 7209473, upload-time = "2026-11-18T04:27:45.583Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6f/a46b468fac256ee6f301ad078c4e8738e2d492795fe298179a39e04afdcb/grpcio-1.85.3-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:53443ce822ace2723a27a1d94d7ae4c9a49fbab888ff9bad9c501b8d6cdc6339", size = 7510210, upload-time = "2026-11-18T04:27:47.489Z" }, + { url = "https://files.pythonhosted.org/packages/d8/2a/6a6641ecd03615d840bd819feddd46e76eb1fccf47d91fb015a8255a3046/grpcio-1.85.3-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b80182b312759d0eebd7aa4dc13ad2ba2de0441398dbc25a5d1aa8e0c0101070", size = 7409583, upload-time = "2026-11-18T04:27:49.306Z" }, + { url = "https://files.pythonhosted.org/packages/98/30/742e190fd71466b72cb502789fa3afdfd077cce7c87f2fcbab644d5d8a9b/grpcio-1.85.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:f047dc3ab821621c86f5c99cd376414681d8e00dfa6a98d01ba68fbabbeae1f5", size = 7962909, upload-time = "2026-11-18T04:27:51.366Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b7/0542a74f844f8b9ea59b969ed19563dd0f3ed0dc512aade29360fe45b363/grpcio-1.85.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:42324e16026c9d6c817718e5a471cc37bd99401bba6a8078243b0f6ab03d13a1", size = 8501815, upload-time = "2026-11-18T04:27:53.621Z" }, + { url = "https://files.pythonhosted.org/packages/49/a1/3fcd7edce1e828d7bee4a830f7f7982884b26ca0da140473443977f9f2b1/grpcio-1.85.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:0aa95286ed10b23c71f3179bad4a52c7ba92a6624c306511876d530f92c951a0", size = 8345782, upload-time = "2026-11-18T04:27:55.432Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a4/6ef6fc206a3ab51f30186d882034216e4fa3af6aebc10fbeb2d8a7caab4b/grpcio-1.85.3-cp315-cp315-win32.whl", hash = "sha256:df6de234f7ee8b65177ef731decc88495dbed42446c4cf007a411280ffbda86c", size = 4854418, upload-time = "2026-11-18T04:27:56.957Z" }, + { url = "https://files.pythonhosted.org/packages/2e/59/e55fc8322982b0dd54dabfc7660d56e5dd1bf6c5ecfbe84c556b4d826365/grpcio-1.85.3-cp315-cp315-win_amd64.whl", hash = "sha256:db5a410f61d4030a1d605e482095c216e3ff107571d37f2de5005c23d336b7ef", size = 5758673, upload-time = "2026-11-18T04:27:58.474Z" }, +] + +[[package]] +name = "protobuf" +version = "7.36.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/89/5b8517baa72f84a67b8a307ba953c91057af618bf40bf676f3c03551f8f0/protobuf-7.36.2.tar.gz", hash = "sha256:497d0463ff3316681da6c0b9e8d06cb465d61abce00b613ab42226175644d1bb", size = 512737, upload-time = "2026-09-17T20:07:59.326Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/72/98342feb672507c8f3a69e34b4fa8961f608edba5c1a48a6f47156d92cb5/protobuf-7.36.2-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:cbc70b17ee27e28894c7fee8bb04be1abead49e936bc70eb60052531eee2079e", size = 456039, upload-time = "2026-09-17T20:07:51.542Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ea/91fdf7c2b8bbd49cde056f00a9df6773532987e1c00fe2830b895af95c7e/protobuf-7.36.2-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:e11e1f0180583a2af89db6a2ecd9e8dc40aa6d2988ca175bfd0e6d12ea72d74e", size = 344219, upload-time = "2026-09-17T20:07:52.914Z" }, + { url = "https://files.pythonhosted.org/packages/17/ab/5fd5f8ece73fad885c5a09aa849b32d70472f954ba3a92d3bb5974ea953b/protobuf-7.36.2-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:f4fee11ec330d238b34a05c9b675f693c20415d1c5bd7d5320cc2f8a798eb9cf", size = 357223, upload-time = "2026-09-17T20:07:53.985Z" }, + { url = "https://files.pythonhosted.org/packages/db/f3/3996583dd2906297a637af12114deddf7658af6e683fedb83be061983fb5/protobuf-7.36.2-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2", size = 343223, upload-time = "2026-09-17T20:07:54.931Z" }, + { url = "https://files.pythonhosted.org/packages/fc/1b/dcc64f358fcb51811b58ae40b3d28f820725f116d86487cc20bd4b130701/protobuf-7.36.2-cp310-abi3-win32.whl", hash = "sha256:912c1221170e16c08d1f086762f563dd61ff83c18b5fa6652952dfaded66f728", size = 442998, upload-time = "2026-09-17T20:07:55.826Z" }, + { url = "https://files.pythonhosted.org/packages/8a/55/b77bda4e5e5f5971fb51b07663694690e9afdb9402136c16a522bd621cad/protobuf-7.36.2-cp310-abi3-win_amd64.whl", hash = "sha256:a300819d441e078a5608c0d3c709796bb548136058fda017ae51d425b44fd353", size = 456514, upload-time = "2026-09-17T20:07:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/d52c7016b04b6c5108f26691f9d33ec82a9b65d041f1a9c771137693d618/protobuf-7.36.2-py3-none-any.whl", hash = "sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e", size = 179806, upload-time = "2026-09-17T20:07:58.211Z" }, +] + [[package]] name = "pydantic" version = "2.13.5" @@ -106,11 +172,17 @@ name = "trial-eligibility-python" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, { name = "pydantic" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = ">=2.9" }] +requires-dist = [ + { name = "grpcio", specifier = ">=1.85" }, + { name = "protobuf", specifier = ">=7.35,<8" }, + { name = "pydantic", specifier = ">=2.9" }, +] [[package]] name = "typing-extensions" diff --git a/secure/sce-unit/Dockerfile b/secure/sce-unit/Dockerfile index 67dee608..add46cb6 100644 --- a/secure/sce-unit/Dockerfile +++ b/secure/sce-unit/Dockerfile @@ -1,10 +1,11 @@ -FROM golang:1.25-alpine AS build +FROM golang:1.26-alpine AS build WORKDIR /src/secure/sce-unit RUN apk add --no-cache ca-certificates COPY packages/go-common /src/packages/go-common COPY secure/sce-unit/go.mod secure/sce-unit/go.sum ./ RUN go mod download COPY secure/sce-unit/*.go ./ +COPY secure/sce-unit/gen ./gen RUN CGO_ENABLED=0 go build -o /sce-unit . FROM scratch diff --git a/secure/sce-unit/README.md b/secure/sce-unit/README.md index a6245c85..13006b77 100644 --- a/secure/sce-unit/README.md +++ b/secure/sce-unit/README.md @@ -1,15 +1,16 @@ # sce-unit The composed image's real entrypoint. Runs as root, requests a single-use verifier -challenge, attests to the Verifier Service with that challenge as the NSM nonce, -fetches and decrypts -PDM records, writes `records.json` under the DC rootfs `/data` (JSON shape -`algorithmId` + `records`, same load path as rads), starts nsjail with -`RECORDS_PATH` / `RESULT_PATH` set, reads the DC app's `result.json`, then streams -that result to the parent instance over vsock (CID 3, port 5000) - the enclave's -only channel out. +challenge, starts the jailed Data Controller app, then serves it over an AF_UNIX +gRPC socket at `/data/sce.sock`. The app calls `FetchRecords`, which triggers +attestation to the Verifier Service with that challenge as the NSM nonce, fetches +the algorithm's PDM records, unwraps the data key via the KMS Service, decrypts +each protobuf `RecordEnvelope`, and streams `sce.v1.Record` messages back. The app +computes and calls `SubmitResult` with one `google.protobuf.Struct`; sce-unit +converts that result to JSON and streams it to the parent instance over vsock +(CID 3, port 5000) - the enclave's only channel out. PDM returns a standalone `dataSubjectIds` list aligned with the records. sce-unit -keeps those ids in memory, writes `records.json` without them, and after a +keeps those ids in memory, never exposes them to the DC app, and after a successful DC app run posts the list to log-service `POST /subject-access` with its attested token. Run-timeline events go to telemetry-service instead, without a token. The DC app never sees subject ids. diff --git a/secure/sce-unit/crypto.go b/secure/sce-unit/crypto.go index f9c9a879..554abb63 100644 --- a/secure/sce-unit/crypto.go +++ b/secure/sce-unit/crypto.go @@ -4,31 +4,24 @@ import ( "crypto/aes" "crypto/cipher" "encoding/base64" - "encoding/json" "fmt" -) -// recordEnvelope is the shape pdm-service stores each record's Data as: AES-256-GCM ciphertext of the -// original JSON, with the nonce and auth tag alongside. The data key that decrypts it is unwrapped via KMS. -type recordEnvelope struct { - V string `json:"v"` - Nonce string `json:"nonce"` - CT string `json:"ct"` -} + scev1 "sce-unit/gen/scev1" -// decryptRecord decrypts one envelope under the plaintext data key and returns the original record object. -func decryptRecord(dek []byte, blob json.RawMessage) (json.RawMessage, error) { - var env recordEnvelope - if err := json.Unmarshal(blob, &env); err != nil { - return nil, fmt.Errorf("parse envelope: %w", err) - } - nonce, err := base64.StdEncoding.DecodeString(env.Nonce) + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" +) + +// decryptRecord decrypts one RecordEnvelope under the plaintext data key. pdm-service sends the envelope as +// base64, and the ciphertext is AES-256-GCM over a serialized google.protobuf.Struct. +func decryptRecord(dek []byte, envelopeB64 string) (*structpb.Struct, error) { + raw, err := base64.StdEncoding.DecodeString(envelopeB64) if err != nil { - return nil, fmt.Errorf("decode nonce: %w", err) + return nil, fmt.Errorf("decode envelope: %w", err) } - body, err := base64.StdEncoding.DecodeString(env.CT) - if err != nil { - return nil, fmt.Errorf("decode ciphertext: %w", err) + envelope := &scev1.RecordEnvelope{} + if err := proto.Unmarshal(raw, envelope); err != nil { + return nil, fmt.Errorf("parse envelope: %w", err) } block, err := aes.NewCipher(dek) @@ -39,13 +32,18 @@ func decryptRecord(dek []byte, blob json.RawMessage) (json.RawMessage, error) { if err != nil { return nil, err } - if len(body) < gcm.Overhead() { + if len(envelope.GetCiphertext()) < gcm.Overhead() { return nil, fmt.Errorf("ciphertext too short") } // Go's GCM appends the auth tag to the ciphertext, matching how seeding built the envelope. - plain, err := gcm.Open(nil, nonce, body, nil) + plain, err := gcm.Open(nil, envelope.GetNonce(), envelope.GetCiphertext(), nil) if err != nil { return nil, fmt.Errorf("decrypt record: %w", err) } - return plain, nil + + data := &structpb.Struct{} + if err := proto.Unmarshal(plain, data); err != nil { + return nil, fmt.Errorf("parse record data: %w", err) + } + return data, nil } diff --git a/secure/sce-unit/gen/scev1/envelope.pb.go b/secure/sce-unit/gen/scev1/envelope.pb.go new file mode 100644 index 00000000..06627d8f --- /dev/null +++ b/secure/sce-unit/gen/scev1/envelope.pb.go @@ -0,0 +1,144 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.6 +// protoc v7.35.1 +// source: sce/v1/envelope.proto + +package scev1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// RecordEnvelope is what pdm-service stores in DynamoDB Data. ciphertext is AES-256-GCM over a serialized +// sce.v1.Record; nonce is the 96-bit GCM nonce, and the auth tag is appended to ciphertext (Go GCM layout). +type RecordEnvelope struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version uint32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + Nonce []byte `protobuf:"bytes,2,opt,name=nonce,proto3" json:"nonce,omitempty"` + Ciphertext []byte `protobuf:"bytes,3,opt,name=ciphertext,proto3" json:"ciphertext,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RecordEnvelope) Reset() { + *x = RecordEnvelope{} + mi := &file_sce_v1_envelope_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RecordEnvelope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RecordEnvelope) ProtoMessage() {} + +func (x *RecordEnvelope) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_envelope_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RecordEnvelope.ProtoReflect.Descriptor instead. +func (*RecordEnvelope) Descriptor() ([]byte, []int) { + return file_sce_v1_envelope_proto_rawDescGZIP(), []int{0} +} + +func (x *RecordEnvelope) GetVersion() uint32 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *RecordEnvelope) GetNonce() []byte { + if x != nil { + return x.Nonce + } + return nil +} + +func (x *RecordEnvelope) GetCiphertext() []byte { + if x != nil { + return x.Ciphertext + } + return nil +} + +var File_sce_v1_envelope_proto protoreflect.FileDescriptor + +const file_sce_v1_envelope_proto_rawDesc = "" + + "\n" + + "\x15sce/v1/envelope.proto\x12\x06sce.v1\"`\n" + + "\x0eRecordEnvelope\x12\x18\n" + + "\aversion\x18\x01 \x01(\rR\aversion\x12\x14\n" + + "\x05nonce\x18\x02 \x01(\fR\x05nonce\x12\x1e\n" + + "\n" + + "ciphertext\x18\x03 \x01(\fR\n" + + "ciphertextb\x06proto3" + +var ( + file_sce_v1_envelope_proto_rawDescOnce sync.Once + file_sce_v1_envelope_proto_rawDescData []byte +) + +func file_sce_v1_envelope_proto_rawDescGZIP() []byte { + file_sce_v1_envelope_proto_rawDescOnce.Do(func() { + file_sce_v1_envelope_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sce_v1_envelope_proto_rawDesc), len(file_sce_v1_envelope_proto_rawDesc))) + }) + return file_sce_v1_envelope_proto_rawDescData +} + +var file_sce_v1_envelope_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_sce_v1_envelope_proto_goTypes = []any{ + (*RecordEnvelope)(nil), // 0: sce.v1.RecordEnvelope +} +var file_sce_v1_envelope_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_sce_v1_envelope_proto_init() } +func file_sce_v1_envelope_proto_init() { + if File_sce_v1_envelope_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_sce_v1_envelope_proto_rawDesc), len(file_sce_v1_envelope_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_sce_v1_envelope_proto_goTypes, + DependencyIndexes: file_sce_v1_envelope_proto_depIdxs, + MessageInfos: file_sce_v1_envelope_proto_msgTypes, + }.Build() + File_sce_v1_envelope_proto = out.File + file_sce_v1_envelope_proto_goTypes = nil + file_sce_v1_envelope_proto_depIdxs = nil +} diff --git a/secure/sce-unit/gen/scev1/records.pb.go b/secure/sce-unit/gen/scev1/records.pb.go new file mode 100644 index 00000000..b4a17f7a --- /dev/null +++ b/secure/sce-unit/gen/scev1/records.pb.go @@ -0,0 +1,268 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.6 +// protoc v7.35.1 +// source: sce/v1/records.proto + +package scev1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + structpb "google.golang.org/protobuf/types/known/structpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Record is one personal-data record handed to the DC application. record_id stays separate so the DC app +// never sees data subject ids; data is schemaless per dataset, so it travels as a Struct. +type Record struct { + state protoimpl.MessageState `protogen:"open.v1"` + RecordId string `protobuf:"bytes,1,opt,name=record_id,json=recordId,proto3" json:"record_id,omitempty"` + Data *structpb.Struct `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Record) Reset() { + *x = Record{} + mi := &file_sce_v1_records_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Record) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Record) ProtoMessage() {} + +func (x *Record) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_records_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Record.ProtoReflect.Descriptor instead. +func (*Record) Descriptor() ([]byte, []int) { + return file_sce_v1_records_proto_rawDescGZIP(), []int{0} +} + +func (x *Record) GetRecordId() string { + if x != nil { + return x.RecordId + } + return "" +} + +func (x *Record) GetData() *structpb.Struct { + if x != nil { + return x.Data + } + return nil +} + +type FetchRecordsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FetchRecordsRequest) Reset() { + *x = FetchRecordsRequest{} + mi := &file_sce_v1_records_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FetchRecordsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchRecordsRequest) ProtoMessage() {} + +func (x *FetchRecordsRequest) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_records_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchRecordsRequest.ProtoReflect.Descriptor instead. +func (*FetchRecordsRequest) Descriptor() ([]byte, []int) { + return file_sce_v1_records_proto_rawDescGZIP(), []int{1} +} + +type SubmitResultRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Result *structpb.Struct `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubmitResultRequest) Reset() { + *x = SubmitResultRequest{} + mi := &file_sce_v1_records_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubmitResultRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubmitResultRequest) ProtoMessage() {} + +func (x *SubmitResultRequest) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_records_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubmitResultRequest.ProtoReflect.Descriptor instead. +func (*SubmitResultRequest) Descriptor() ([]byte, []int) { + return file_sce_v1_records_proto_rawDescGZIP(), []int{2} +} + +func (x *SubmitResultRequest) GetResult() *structpb.Struct { + if x != nil { + return x.Result + } + return nil +} + +type SubmitResultResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubmitResultResponse) Reset() { + *x = SubmitResultResponse{} + mi := &file_sce_v1_records_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubmitResultResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubmitResultResponse) ProtoMessage() {} + +func (x *SubmitResultResponse) ProtoReflect() protoreflect.Message { + mi := &file_sce_v1_records_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubmitResultResponse.ProtoReflect.Descriptor instead. +func (*SubmitResultResponse) Descriptor() ([]byte, []int) { + return file_sce_v1_records_proto_rawDescGZIP(), []int{3} +} + +var File_sce_v1_records_proto protoreflect.FileDescriptor + +const file_sce_v1_records_proto_rawDesc = "" + + "\n" + + "\x14sce/v1/records.proto\x12\x06sce.v1\x1a\x1cgoogle/protobuf/struct.proto\"R\n" + + "\x06Record\x12\x1b\n" + + "\trecord_id\x18\x01 \x01(\tR\brecordId\x12+\n" + + "\x04data\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x04data\"\x15\n" + + "\x13FetchRecordsRequest\"F\n" + + "\x13SubmitResultRequest\x12/\n" + + "\x06result\x18\x01 \x01(\v2\x17.google.protobuf.StructR\x06result\"\x16\n" + + "\x14SubmitResultResponse2\x96\x01\n" + + "\n" + + "DataAccess\x12=\n" + + "\fFetchRecords\x12\x1b.sce.v1.FetchRecordsRequest\x1a\x0e.sce.v1.Record0\x01\x12I\n" + + "\fSubmitResult\x12\x1b.sce.v1.SubmitResultRequest\x1a\x1c.sce.v1.SubmitResultResponseb\x06proto3" + +var ( + file_sce_v1_records_proto_rawDescOnce sync.Once + file_sce_v1_records_proto_rawDescData []byte +) + +func file_sce_v1_records_proto_rawDescGZIP() []byte { + file_sce_v1_records_proto_rawDescOnce.Do(func() { + file_sce_v1_records_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sce_v1_records_proto_rawDesc), len(file_sce_v1_records_proto_rawDesc))) + }) + return file_sce_v1_records_proto_rawDescData +} + +var file_sce_v1_records_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_sce_v1_records_proto_goTypes = []any{ + (*Record)(nil), // 0: sce.v1.Record + (*FetchRecordsRequest)(nil), // 1: sce.v1.FetchRecordsRequest + (*SubmitResultRequest)(nil), // 2: sce.v1.SubmitResultRequest + (*SubmitResultResponse)(nil), // 3: sce.v1.SubmitResultResponse + (*structpb.Struct)(nil), // 4: google.protobuf.Struct +} +var file_sce_v1_records_proto_depIdxs = []int32{ + 4, // 0: sce.v1.Record.data:type_name -> google.protobuf.Struct + 4, // 1: sce.v1.SubmitResultRequest.result:type_name -> google.protobuf.Struct + 1, // 2: sce.v1.DataAccess.FetchRecords:input_type -> sce.v1.FetchRecordsRequest + 2, // 3: sce.v1.DataAccess.SubmitResult:input_type -> sce.v1.SubmitResultRequest + 0, // 4: sce.v1.DataAccess.FetchRecords:output_type -> sce.v1.Record + 3, // 5: sce.v1.DataAccess.SubmitResult:output_type -> sce.v1.SubmitResultResponse + 4, // [4:6] is the sub-list for method output_type + 2, // [2:4] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_sce_v1_records_proto_init() } +func file_sce_v1_records_proto_init() { + if File_sce_v1_records_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_sce_v1_records_proto_rawDesc), len(file_sce_v1_records_proto_rawDesc)), + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_sce_v1_records_proto_goTypes, + DependencyIndexes: file_sce_v1_records_proto_depIdxs, + MessageInfos: file_sce_v1_records_proto_msgTypes, + }.Build() + File_sce_v1_records_proto = out.File + file_sce_v1_records_proto_goTypes = nil + file_sce_v1_records_proto_depIdxs = nil +} diff --git a/secure/sce-unit/gen/scev1/records_grpc.pb.go b/secure/sce-unit/gen/scev1/records_grpc.pb.go new file mode 100644 index 00000000..2e22911a --- /dev/null +++ b/secure/sce-unit/gen/scev1/records_grpc.pb.go @@ -0,0 +1,171 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v7.35.1 +// source: sce/v1/records.proto + +package scev1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + DataAccess_FetchRecords_FullMethodName = "/sce.v1.DataAccess/FetchRecords" + DataAccess_SubmitResult_FullMethodName = "/sce.v1.DataAccess/SubmitResult" +) + +// DataAccessClient is the client API for DataAccess service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. +type DataAccessClient interface { + // Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. + FetchRecords(ctx context.Context, in *FetchRecordsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Record], error) + // The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. + SubmitResult(ctx context.Context, in *SubmitResultRequest, opts ...grpc.CallOption) (*SubmitResultResponse, error) +} + +type dataAccessClient struct { + cc grpc.ClientConnInterface +} + +func NewDataAccessClient(cc grpc.ClientConnInterface) DataAccessClient { + return &dataAccessClient{cc} +} + +func (c *dataAccessClient) FetchRecords(ctx context.Context, in *FetchRecordsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Record], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &DataAccess_ServiceDesc.Streams[0], DataAccess_FetchRecords_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[FetchRecordsRequest, Record]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type DataAccess_FetchRecordsClient = grpc.ServerStreamingClient[Record] + +func (c *dataAccessClient) SubmitResult(ctx context.Context, in *SubmitResultRequest, opts ...grpc.CallOption) (*SubmitResultResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SubmitResultResponse) + err := c.cc.Invoke(ctx, DataAccess_SubmitResult_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// DataAccessServer is the server API for DataAccess service. +// All implementations must embed UnimplementedDataAccessServer +// for forward compatibility. +// +// DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. +type DataAccessServer interface { + // Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. + FetchRecords(*FetchRecordsRequest, grpc.ServerStreamingServer[Record]) error + // The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. + SubmitResult(context.Context, *SubmitResultRequest) (*SubmitResultResponse, error) + mustEmbedUnimplementedDataAccessServer() +} + +// UnimplementedDataAccessServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedDataAccessServer struct{} + +func (UnimplementedDataAccessServer) FetchRecords(*FetchRecordsRequest, grpc.ServerStreamingServer[Record]) error { + return status.Errorf(codes.Unimplemented, "method FetchRecords not implemented") +} +func (UnimplementedDataAccessServer) SubmitResult(context.Context, *SubmitResultRequest) (*SubmitResultResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SubmitResult not implemented") +} +func (UnimplementedDataAccessServer) mustEmbedUnimplementedDataAccessServer() {} +func (UnimplementedDataAccessServer) testEmbeddedByValue() {} + +// UnsafeDataAccessServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to DataAccessServer will +// result in compilation errors. +type UnsafeDataAccessServer interface { + mustEmbedUnimplementedDataAccessServer() +} + +func RegisterDataAccessServer(s grpc.ServiceRegistrar, srv DataAccessServer) { + // If the following call pancis, it indicates UnimplementedDataAccessServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&DataAccess_ServiceDesc, srv) +} + +func _DataAccess_FetchRecords_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(FetchRecordsRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(DataAccessServer).FetchRecords(m, &grpc.GenericServerStream[FetchRecordsRequest, Record]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type DataAccess_FetchRecordsServer = grpc.ServerStreamingServer[Record] + +func _DataAccess_SubmitResult_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SubmitResultRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DataAccessServer).SubmitResult(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DataAccess_SubmitResult_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DataAccessServer).SubmitResult(ctx, req.(*SubmitResultRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// DataAccess_ServiceDesc is the grpc.ServiceDesc for DataAccess service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var DataAccess_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "sce.v1.DataAccess", + HandlerType: (*DataAccessServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SubmitResult", + Handler: _DataAccess_SubmitResult_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "FetchRecords", + Handler: _DataAccess_FetchRecords_Handler, + ServerStreams: true, + }, + }, + Metadata: "sce/v1/records.proto", +} diff --git a/secure/sce-unit/go.mod b/secure/sce-unit/go.mod index 81120465..d1af40d8 100644 --- a/secure/sce-unit/go.mod +++ b/secure/sce-unit/go.mod @@ -1,13 +1,15 @@ module sce-unit -go 1.25.0 +go 1.26.0 require ( github.com/Secure-RADS/st-dp/packages/go-common v0.0.0 github.com/edgebitio/nitro-enclaves-sdk-go v1.0.0 github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 github.com/mdlayher/vsock v1.3.0 - golang.org/x/sys v0.45.0 + golang.org/x/sys v0.48.0 + google.golang.org/grpc v1.85.0 + google.golang.org/protobuf v1.36.12 ) replace github.com/Secure-RADS/st-dp/packages/go-common => ../../packages/go-common @@ -16,6 +18,8 @@ require ( github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/mdlayher/socket v0.6.0 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sync v0.20.0 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect ) diff --git a/secure/sce-unit/go.sum b/secure/sce-unit/go.sum index 602b5df0..65686c44 100644 --- a/secure/sce-unit/go.sum +++ b/secure/sce-unit/go.sum @@ -5,6 +5,8 @@ github.com/edgebitio/nitro-enclaves-sdk-go v1.0.0/go.mod h1:hDAX5hYfgVR/TzewAODj github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 h1:pU32bJGmZwF4WXb9Yaz0T8vHDtIPVxqDOdmYdwTQPqw= @@ -29,19 +31,21 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -49,5 +53,13 @@ golang.org/x/tools v0.0.0-20210105210202-9ed45478a130/go.mod h1:emZCQorbCU4vsT4f golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/secure/sce-unit/main.go b/secure/sce-unit/main.go index 1dd91e10..2972c48a 100644 --- a/secure/sce-unit/main.go +++ b/secure/sce-unit/main.go @@ -12,16 +12,16 @@ import ( sharedlog "github.com/Secure-RADS/st-dp/packages/go-common/log" "golang.org/x/sys/unix" + "google.golang.org/protobuf/encoding/protojson" ) const ( nsjailBin = "/bin/nsjail" nsjailCfg = "/etc/sce/nsjail.cfg" - // Host paths under the DC rootfs; nsjail bind-mounts that rootfs at "/", so the jail sees /data/... - dataDirHost = "/opt/dc-app/rootfs/data" - recordsFileHost = dataDirHost + "/records.json" - resultFileHost = dataDirHost + "/result.json" + // Host path to the DC rootfs; nsjail bind-mounts it at "/", so the jail sees /data/sce.sock. + dataDirHost = "/opt/dc-app/rootfs/data" + rpcSocketHost = dataDirHost + "/sce.sock" // The parent EC2 instance is always CID 3 on Nitro; the host's user data listens on this port. hostCID = 3 @@ -63,8 +63,14 @@ func main() { log.Printf("sce-unit: seeded kernel entropy from NSM") timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventUnitReady, nil) - // Attest, fetch, decrypt, and write records.json before nsjail starts - same file contract as rads. - subjectIDs, err := prepareRecordsFile(cfg, timelineLog, runID, info.Size) + // The DC app pulls records over gRPC, so the socket must exist before the jail starts. + if err := os.MkdirAll(dataDirHost, 0o755); err != nil { + failRun(timelineLog, cfg.AlgorithmId, runID, fmt.Errorf("create %s: %w", dataDirHost, err)) + } + _ = os.Remove(rpcSocketHost) + + server := newDataAccessServer(cfg, timelineLog, runID, info.Size) + grpcServer, listener, err := serveRPC(rpcSocketHost, server) if err != nil { failRun(timelineLog, cfg.AlgorithmId, runID, err) } @@ -74,52 +80,59 @@ func main() { cmd.Stdout = io.MultiWriter(os.Stdout, &jailLog) cmd.Stderr = io.MultiWriter(os.Stderr, &jailLog) if err := cmd.Start(); err != nil { + grpcServer.Stop() + listener.Close() failRun(timelineLog, cfg.AlgorithmId, runID, fmt.Errorf("start nsjail: %w", err)) } log.Printf("sce-unit: nsjail started (pid %d)", cmd.Process.Pid) waitErr := cmd.Wait() log.Printf("sce-unit: nsjail exited: %v", waitErr) + grpcServer.Stop() + listener.Close() - output, readErr := os.ReadFile(resultFileHost) - - failed := waitErr != nil || readErr != nil || len(output) == 0 + result := server.submittedResult() + failed := waitErr != nil || result == nil if failed { timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventRunFailed, map[string]any{ - "error": fmt.Sprintf("dc app failed: waitErr=%v readErr=%v outputBytes=%d", waitErr, readErr, len(output)), - "outputBytes": len(output), + "error": fmt.Sprintf("dc app failed: waitErr=%v resultSubmitted=%t", waitErr, result != nil), }) - } else { - timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventDCAppCompleted, map[string]any{"outputBytes": len(output)}) - // DSAR trail: record the data subjects the app used only after it ran successfully, so a run that - // never reached the DC app does not show up as an access. - accessLog.EmitSubjectAccess(cfg.AlgorithmId, runID, subjectIDs) - } - if readErr != nil || len(output) == 0 { - // The enclave console is lost when the host terminates, so deliver the failure and the jailed app's own output. - log.Printf("sce-unit: result missing or empty: readErr=%v bytes=%d waitErr=%v", readErr, len(output), waitErr) failure, _ := json.Marshal(struct { Error string `json:"error"` WaitErr string `json:"waitErr"` JailLog string `json:"jailLog"` }{ - Error: fmt.Sprintf("dc app produced no result file: %v", readErr), + Error: "dc app produced no result", WaitErr: fmt.Sprintf("%v", waitErr), JailLog: jailLog.String(), }) - output = append(failure, '\n') - } else if output[len(output)-1] != '\n' { - output = append(output, '\n') + output := append(failure, '\n') + if err := sendOutput(output); err != nil { + log.Printf("sce-unit: send output to host failed: %v", err) + timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventRunFailed, map[string]any{"error": err.Error()}) + os.Exit(1) + } + timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventOutputSent, nil) + os.Exit(1) + } + + output, err := protojson.Marshal(result) + if err != nil { + failRun(timelineLog, cfg.AlgorithmId, runID, fmt.Errorf("encode result: %w", err)) } + output = append(output, '\n') + + timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventDCAppCompleted, map[string]any{"outputBytes": len(output)}) + // DSAR trail: record the data subjects the app used only after it ran successfully, so a run that + // never reached the DC app does not show up as an access. + accessLog.EmitSubjectAccess(cfg.AlgorithmId, runID, server.dataSubjects()) + if err := sendOutput(output); err != nil { log.Printf("sce-unit: send output to host failed: %v", err) timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventRunFailed, map[string]any{"error": err.Error()}) os.Exit(1) } timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventOutputSent, nil) - if failed { - os.Exit(1) - } } func failRun(timelineLog *sharedlog.Client, algorithmID, runID string, err error) { @@ -136,68 +149,7 @@ func failRun(timelineLog *sharedlog.Client, algorithmID, runID string, err error os.Exit(1) } -// prepareRecordsFile attests, fetches ciphertext from pdm-service, decrypts, and writes the rads-shaped -// records.json the DC app loads via RECORDS_PATH. It also returns the data subjects whose records were -// handed to the DC app, for the post-completion subject-access trail. -func prepareRecordsFile(cfg runConfig, timelineLog *sharedlog.Client, runID string, limit int) ([]string, error) { - tp, err := attest(cfg, runID) - if err != nil { - return nil, fmt.Errorf("attestation failed: %w", err) - } - log.Printf("sce-unit: attested successfully, access token expires %s", tp.AccessTokenExpiresAt.Format(time.RFC3339)) - setTokens(tp) - go refreshLoop(cfg, runID) - // Every event from here on carries the token. - timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventAttested, nil) - - records, subjectIDs, err := fetchAllRecords(cfg, limit) - if err != nil { - return nil, fmt.Errorf("fetch records: %w", err) - } - log.Printf("sce-unit: fetched %d records from pdm-service", len(records)) - timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventRecordsFetched, map[string]any{"count": len(records)}) - - // pdm-service only ever returns ciphertext; kms-service unwraps the data key under this enclave's - // attestation. Only the measured image can do this - see the per-algorithm key policy. - dek, err := decryptDataKey(cfg) - if err != nil { - return nil, fmt.Errorf("decrypt data key: %w", err) - } - log.Printf("sce-unit: unwrapped data key via kms-service (%d bytes)", len(dek)) - - plainRecords := make([]pdmRecord, 0, len(records)) - for _, rec := range records { - plain, derr := decryptRecord(dek, rec.Data) - if derr != nil { - return nil, fmt.Errorf("decrypt record: %w", derr) - } - plainRecords = append(plainRecords, pdmRecord{RecordId: rec.RecordId, Data: plain}) - } - subjectIDs = uniqueIDs(subjectIDs) - log.Printf("sce-unit: decrypted %d records", len(plainRecords)) - timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventRecordsDecrypted, map[string]any{"count": len(plainRecords)}) - - body, err := json.Marshal(struct { - AlgorithmId string `json:"algorithmId"` - Records []pdmRecord `json:"records"` - }{AlgorithmId: cfg.AlgorithmId, Records: plainRecords}) - if err != nil { - return nil, fmt.Errorf("marshal records file: %w", err) - } - if err := os.MkdirAll(dataDirHost, 0o755); err != nil { - return nil, fmt.Errorf("create %s: %w", dataDirHost, err) - } - // Drop a leftover result from a prior attempt in this rootfs (fresh EIFs are clean; defensive). - _ = os.Remove(resultFileHost) - if err := os.WriteFile(recordsFileHost, body, 0o644); err != nil { - return nil, fmt.Errorf("write %s: %w", recordsFileHost, err) - } - log.Printf("sce-unit: wrote %d bytes to %s", len(body), recordsFileHost) - timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventRecordsStoredLocally, nil) - return subjectIDs, nil -} - -// sendOutput streams the DC app's response to the parent instance over vsock - the enclave's only channel out. +// sendOutput streams the DC app's result to the parent instance over vsock - the enclave's only channel out. func sendOutput(output []byte) error { fd, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM, 0) if err != nil { diff --git a/secure/sce-unit/pdm.go b/secure/sce-unit/pdm.go index 76f50e62..3c874a8a 100644 --- a/secure/sce-unit/pdm.go +++ b/secure/sce-unit/pdm.go @@ -13,8 +13,9 @@ import ( const pdmRequestTimeout = 60 * time.Second type pdmRecord struct { - RecordId string `json:"recordId"` - Data json.RawMessage `json:"data"` + RecordId string `json:"recordId"` + // Data is the base64 RecordEnvelope pdm-service reads from DynamoDB. + Data string `json:"data"` } // fetchAllRecords pulls personal-data records for this enclave's algorithm from the PDM service, following diff --git a/secure/sce-unit/rpc.go b/secure/sce-unit/rpc.go new file mode 100644 index 00000000..b5251c43 --- /dev/null +++ b/secure/sce-unit/rpc.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "fmt" + "log" + "net" + "sync" + "time" + + scev1 "sce-unit/gen/scev1" + + sharedlog "github.com/Secure-RADS/st-dp/packages/go-common/log" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" +) + +// dataAccessServer serves the DC app over an AF_UNIX socket inside the enclave. Attestation, record fetch, +// and decryption are triggered by the app's FetchRecords call rather than run before nsjail starts. +type dataAccessServer struct { + scev1.UnimplementedDataAccessServer + + cfg runConfig + timelineLog *sharedlog.Client + runID string + limit int + + fetchOnce sync.Once + records []*scev1.Record + subjectIDs []string + fetchErr error + + subjectMu sync.Mutex + submitOnce sync.Once + submitted chan struct{} + result *structpb.Struct +} + +func newDataAccessServer(cfg runConfig, timelineLog *sharedlog.Client, runID string, limit int) *dataAccessServer { + return &dataAccessServer{ + cfg: cfg, + timelineLog: timelineLog, + runID: runID, + limit: limit, + submitted: make(chan struct{}), + } +} + +func (s *dataAccessServer) FetchRecords(_ *scev1.FetchRecordsRequest, stream scev1.DataAccess_FetchRecordsServer) error { + records, err := s.fetch() + if err != nil { + return status.Errorf(codes.Internal, "fetch records: %v", err) + } + for _, record := range records { + if err := stream.Send(record); err != nil { + return err + } + } + return nil +} + +func (s *dataAccessServer) SubmitResult(_ context.Context, req *scev1.SubmitResultRequest) (*scev1.SubmitResultResponse, error) { + if req.GetResult() == nil { + return nil, status.Error(codes.InvalidArgument, "result is empty") + } + s.submitOnce.Do(func() { + s.result = req.GetResult() + close(s.submitted) + }) + return &scev1.SubmitResultResponse{}, nil +} + +func (s *dataAccessServer) submittedResult() *structpb.Struct { + select { + case <-s.submitted: + return s.result + default: + return nil + } +} + +// fetch runs at most once: the first FetchRecords triggers attestation and pulls the dataset, and retries +// reuse the outcome so a reconnecting client cannot fetch twice. +func (s *dataAccessServer) fetch() ([]*scev1.Record, error) { + s.fetchOnce.Do(func() { + records, subjectIDs, err := s.loadRecords() + s.records, s.fetchErr = records, err + s.subjectMu.Lock() + s.subjectIDs = subjectIDs + s.subjectMu.Unlock() + }) + return s.records, s.fetchErr +} + +func (s *dataAccessServer) loadRecords() ([]*scev1.Record, []string, error) { + tp, err := attest(s.cfg, s.runID) + if err != nil { + return nil, nil, fmt.Errorf("attestation failed: %w", err) + } + log.Printf("sce-unit: attested successfully, access token expires %s", tp.AccessTokenExpiresAt.Format(time.RFC3339)) + setTokens(tp) + go refreshLoop(s.cfg, s.runID) + s.timelineLog.EmitEvent(s.cfg.AlgorithmId, &s.runID, sharedlog.EventAttested, nil) + + records, subjectIDs, err := fetchAllRecords(s.cfg, s.limit) + if err != nil { + return nil, nil, fmt.Errorf("fetch records: %w", err) + } + log.Printf("sce-unit: fetched %d records from pdm-service", len(records)) + s.timelineLog.EmitEvent(s.cfg.AlgorithmId, &s.runID, sharedlog.EventRecordsFetched, map[string]any{"count": len(records)}) + + dek, err := decryptDataKey(s.cfg) + if err != nil { + return nil, nil, fmt.Errorf("decrypt data key: %w", err) + } + log.Printf("sce-unit: unwrapped data key via kms-service (%d bytes)", len(dek)) + + out := make([]*scev1.Record, 0, len(records)) + for _, rec := range records { + data, derr := decryptRecord(dek, rec.Data) + if derr != nil { + return nil, nil, fmt.Errorf("decrypt record %s: %w", rec.RecordId, derr) + } + out = append(out, &scev1.Record{RecordId: rec.RecordId, Data: data}) + } + s.timelineLog.EmitEvent(s.cfg.AlgorithmId, &s.runID, sharedlog.EventRecordsDecrypted, map[string]any{"count": len(out)}) + s.timelineLog.EmitEvent(s.cfg.AlgorithmId, &s.runID, sharedlog.EventRecordsStoredLocally, nil) + + return out, uniqueIDs(subjectIDs), nil +} + +func (s *dataAccessServer) dataSubjects() []string { + s.subjectMu.Lock() + defer s.subjectMu.Unlock() + return s.subjectIDs +} + +// serveRPC listens on the unix socket the jailed DC app connects to and returns the running server. +func serveRPC(socketPath string, server *dataAccessServer) (*grpc.Server, net.Listener, error) { + listener, err := net.Listen("unix", socketPath) + if err != nil { + return nil, nil, fmt.Errorf("listen on %s: %w", socketPath, err) + } + grpcServer := grpc.NewServer() + scev1.RegisterDataAccessServer(grpcServer, server) + go func() { + if err := grpcServer.Serve(listener); err != nil { + log.Printf("sce-unit: grpc server stopped: %v", err) + } + }() + log.Printf("sce-unit: serving DataAccess on %s", socketPath) + return grpcServer, listener, nil +} diff --git a/seeding/bun.lock b/seeding/bun.lock index 4a494bf1..66ec53a2 100644 --- a/seeding/bun.lock +++ b/seeding/bun.lock @@ -8,6 +8,7 @@ "@aws-sdk/client-dynamodb": "^3.1141.0", "@aws-sdk/client-kms": "^3.1119.0", "@aws-sdk/lib-dynamodb": "^3.700.0", + "@bufbuild/protobuf": "^2.10.0", "p-map": "^7.0.6", "uuid": "^14.0.2", }, @@ -86,6 +87,8 @@ "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.5.10", "", { "os": "win32", "cpu": "x64" }, "sha512-M+2dgBsl3lXRiTfgPVc2p3anS4Tocojke4rzFLScZ2Y/wmF+36dRb1iHCLiyGqOzQGyTplZH1HnEYviiAqi3nA=="], + "@bufbuild/protobuf": ["@bufbuild/protobuf@2.17.0", "", {}, "sha512-hcPkSz6D7MiTU/5VBBhK+kxaYDoOdQ6yJRFJ3kS8Nydp+VZXfg1vrpABKPNod8bXMufsVCmiUJVIKFzCzta4uw=="], + "@faker-js/faker": ["@faker-js/faker@10.6.0", "", {}, "sha512-3RQHgEtvL1Frl/d1cSreo7qhJ3Gk1OdNUai/CtZ8G+wYeRQnJih3s9xJ9/kgYekPQRdwgh0HXRPqMlzWGwivIQ=="], "@smithy/core": ["@smithy/core@3.35.0", "", { "dependencies": { "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-zRMhfkByhT2snNdr1si24vJitU6Cr9ix2MikUfWmkAgp4jrNP0GcKSP5YvwQ+TlI8AZXER5QOGJn3JsVtSD9/A=="], diff --git a/seeding/gen/sce/v1/envelope_pb.ts b/seeding/gen/sce/v1/envelope_pb.ts new file mode 100644 index 00000000..d80dd438 --- /dev/null +++ b/seeding/gen/sce/v1/envelope_pb.ts @@ -0,0 +1,44 @@ +// @generated by protoc-gen-es v2.17.0 with parameter "target=ts" +// @generated from file sce/v1/envelope.proto (package sce.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; + +/** + * Describes the file sce/v1/envelope.proto. + */ +export const file_sce_v1_envelope: GenFile = /*@__PURE__*/ + fileDesc("ChVzY2UvdjEvZW52ZWxvcGUucHJvdG8SBnNjZS52MSJECg5SZWNvcmRFbnZlbG9wZRIPCgd2ZXJzaW9uGAEgASgNEg0KBW5vbmNlGAIgASgMEhIKCmNpcGhlcnRleHQYAyABKAxiBnByb3RvMw"); + +/** + * RecordEnvelope is what pdm-service stores in DynamoDB Data. ciphertext is AES-256-GCM over a serialized + * sce.v1.Record; nonce is the 96-bit GCM nonce, and the auth tag is appended to ciphertext (Go GCM layout). + * + * @generated from message sce.v1.RecordEnvelope + */ +export type RecordEnvelope = Message<"sce.v1.RecordEnvelope"> & { + /** + * @generated from field: uint32 version = 1; + */ + version: number; + + /** + * @generated from field: bytes nonce = 2; + */ + nonce: Uint8Array; + + /** + * @generated from field: bytes ciphertext = 3; + */ + ciphertext: Uint8Array; +}; + +/** + * Describes the message sce.v1.RecordEnvelope. + * Use `create(RecordEnvelopeSchema)` to create a new message. + */ +export const RecordEnvelopeSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_sce_v1_envelope, 0); + diff --git a/seeding/gen/sce/v1/records_pb.ts b/seeding/gen/sce/v1/records_pb.ts new file mode 100644 index 00000000..5bda07e1 --- /dev/null +++ b/seeding/gen/sce/v1/records_pb.ts @@ -0,0 +1,112 @@ +// @generated by protoc-gen-es v2.17.0 with parameter "target=ts" +// @generated from file sce/v1/records.proto (package sce.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import { fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv2"; +import { file_google_protobuf_struct } from "@bufbuild/protobuf/wkt"; +import type { JsonObject, Message } from "@bufbuild/protobuf"; + +/** + * Describes the file sce/v1/records.proto. + */ +export const file_sce_v1_records: GenFile = /*@__PURE__*/ + fileDesc("ChRzY2UvdjEvcmVjb3Jkcy5wcm90bxIGc2NlLnYxIkIKBlJlY29yZBIRCglyZWNvcmRfaWQYASABKAkSJQoEZGF0YRgCIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QiFQoTRmV0Y2hSZWNvcmRzUmVxdWVzdCI+ChNTdWJtaXRSZXN1bHRSZXF1ZXN0EicKBnJlc3VsdBgBIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QiFgoUU3VibWl0UmVzdWx0UmVzcG9uc2UylgEKCkRhdGFBY2Nlc3MSPQoMRmV0Y2hSZWNvcmRzEhsuc2NlLnYxLkZldGNoUmVjb3Jkc1JlcXVlc3QaDi5zY2UudjEuUmVjb3JkMAESSQoMU3VibWl0UmVzdWx0Ehsuc2NlLnYxLlN1Ym1pdFJlc3VsdFJlcXVlc3QaHC5zY2UudjEuU3VibWl0UmVzdWx0UmVzcG9uc2ViBnByb3RvMw", [file_google_protobuf_struct]); + +/** + * Record is one personal-data record handed to the DC application. record_id stays separate so the DC app + * never sees data subject ids; data is schemaless per dataset, so it travels as a Struct. + * + * @generated from message sce.v1.Record + */ +export type Record = Message<"sce.v1.Record"> & { + /** + * @generated from field: string record_id = 1; + */ + recordId: string; + + /** + * @generated from field: google.protobuf.Struct data = 2; + */ + data?: JsonObject | undefined; +}; + +/** + * Describes the message sce.v1.Record. + * Use `create(RecordSchema)` to create a new message. + */ +export const RecordSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_sce_v1_records, 0); + +/** + * @generated from message sce.v1.FetchRecordsRequest + */ +export type FetchRecordsRequest = Message<"sce.v1.FetchRecordsRequest"> & { +}; + +/** + * Describes the message sce.v1.FetchRecordsRequest. + * Use `create(FetchRecordsRequestSchema)` to create a new message. + */ +export const FetchRecordsRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_sce_v1_records, 1); + +/** + * @generated from message sce.v1.SubmitResultRequest + */ +export type SubmitResultRequest = Message<"sce.v1.SubmitResultRequest"> & { + /** + * @generated from field: google.protobuf.Struct result = 1; + */ + result?: JsonObject | undefined; +}; + +/** + * Describes the message sce.v1.SubmitResultRequest. + * Use `create(SubmitResultRequestSchema)` to create a new message. + */ +export const SubmitResultRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_sce_v1_records, 2); + +/** + * @generated from message sce.v1.SubmitResultResponse + */ +export type SubmitResultResponse = Message<"sce.v1.SubmitResultResponse"> & { +}; + +/** + * Describes the message sce.v1.SubmitResultResponse. + * Use `create(SubmitResultResponseSchema)` to create a new message. + */ +export const SubmitResultResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_sce_v1_records, 3); + +/** + * DataAccess runs over an AF_UNIX socket inside the enclave, between sce-unit (server) and the DC app. + * + * @generated from service sce.v1.DataAccess + */ +export const DataAccess: GenService<{ + /** + * Streams this run's decrypted records. sce-unit applies the run-info limit; the request carries no filters. + * + * @generated from rpc sce.v1.DataAccess.FetchRecords + */ + fetchRecords: { + methodKind: "server_streaming"; + input: typeof FetchRecordsRequestSchema; + output: typeof RecordSchema; + }, + /** + * The DC app submits its result once, then exits. sce-unit converts it back to JSON for the host. + * + * @generated from rpc sce.v1.DataAccess.SubmitResult + */ + submitResult: { + methodKind: "unary"; + input: typeof SubmitResultRequestSchema; + output: typeof SubmitResultResponseSchema; + }, +}> = /*@__PURE__*/ + serviceDesc(file_sce_v1_records, 0); + diff --git a/seeding/package.json b/seeding/package.json index 2f4b9156..75491fb6 100644 --- a/seeding/package.json +++ b/seeding/package.json @@ -22,6 +22,7 @@ "@aws-sdk/client-dynamodb": "^3.1141.0", "@aws-sdk/client-kms": "^3.1119.0", "@aws-sdk/lib-dynamodb": "^3.700.0", + "@bufbuild/protobuf": "^2.10.0", "p-map": "^7.0.6", "uuid": "^14.0.2" }, diff --git a/seeding/seed-secure-rads.ts b/seeding/seed-secure-rads.ts index f4d16ee7..bfa2cdfa 100644 --- a/seeding/seed-secure-rads.ts +++ b/seeding/seed-secure-rads.ts @@ -2,6 +2,9 @@ import { createCipheriv, randomBytes } from "node:crypto"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { GenerateDataKeyCommand, KMSClient } from "@aws-sdk/client-kms"; import { GetCommand, PutCommand } from "@aws-sdk/lib-dynamodb"; +import { create, fromJson, type JsonObject, toBinary } from "@bufbuild/protobuf"; +import { StructSchema } from "@bufbuild/protobuf/wkt"; +import { RecordEnvelopeSchema } from "./gen/sce/v1/envelope_pb"; import { readDatasetMeta, resolveSeedNames, streamDatasetRecords } from "./lib/datasets"; import { documentClient, writeAll } from "./lib/dynamo"; import { requireEnv } from "./lib/env"; @@ -23,12 +26,20 @@ function dekCachePath(algorithmId: string): string { return `/tmp/sce-seed-dek-${algorithmId}.bin`; } -function encryptData(dek: Buffer, data: unknown): Record { +function encryptData(dek: Buffer, data: Record): Uint8Array { + const plaintext = toBinary(StructSchema, fromJson(StructSchema, data as JsonObject)); const nonce = randomBytes(12); const cipher = createCipheriv("aes-256-gcm", dek, nonce); - const ct = Buffer.concat([cipher.update(JSON.stringify(data), "utf8"), cipher.final()]); + const ct = Buffer.concat([cipher.update(plaintext), cipher.final()]); const tag = cipher.getAuthTag(); - return { v: "1", nonce: nonce.toString("base64"), ct: Buffer.concat([ct, tag]).toString("base64") }; + return toBinary( + RecordEnvelopeSchema, + create(RecordEnvelopeSchema, { + version: 1, + nonce, + ciphertext: Buffer.concat([ct, tag]), + }), + ); } async function loadOrCreateDek(algorithmId: string, kmsKeyArn: string): Promise { From e3680ec0374b944105c625d1246b9e10ca358e41 Mon Sep 17 00:00:00 2001 From: prdai Date: Fri, 20 Nov 2026 14:17:38 +0530 Subject: [PATCH 2/2] fix(sce): decode Struct payloads in Node and Rust samples Co-authored-by: Codex --- secure/samples/audience-segments-rust/src/main.rs | 11 ++++++++--- secure/samples/product-trends-node/main.js | 11 +++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/secure/samples/audience-segments-rust/src/main.rs b/secure/samples/audience-segments-rust/src/main.rs index 2f0685b8..1d186f8f 100644 --- a/secure/samples/audience-segments-rust/src/main.rs +++ b/secure/samples/audience-segments-rust/src/main.rs @@ -113,9 +113,14 @@ fn value_to_json(value: &prost_types::Value) -> serde_json::Value { use prost_types::value::Kind; match value.kind.as_ref() { Some(Kind::NullValue(_)) => serde_json::Value::Null, - Some(Kind::NumberValue(n)) => serde_json::Number::from_f64(*n) - .map(serde_json::Value::Number) - .unwrap_or(serde_json::Value::Null), + Some(Kind::NumberValue(n)) => { + let number = if n.fract() == 0.0 && *n >= i64::MIN as f64 && *n <= i64::MAX as f64 { + serde_json::Number::from(*n as i64) + } else { + serde_json::Number::from_f64(*n).unwrap_or_else(|| serde_json::Number::from(0)) + }; + serde_json::Value::Number(number) + } Some(Kind::StringValue(s)) => serde_json::Value::String(s.clone()), Some(Kind::BoolValue(b)) => serde_json::Value::Bool(*b), Some(Kind::StructValue(s)) => struct_to_json(s), diff --git a/secure/samples/product-trends-node/main.js b/secure/samples/product-trends-node/main.js index 931addba..56297458 100644 --- a/secure/samples/product-trends-node/main.js +++ b/secure/samples/product-trends-node/main.js @@ -1,6 +1,5 @@ import * as grpc from "@grpc/grpc-js"; -import { create, fromBinary, fromJson, toBinary, toJson } from "@bufbuild/protobuf"; -import { StructSchema } from "@bufbuild/protobuf/wkt"; +import { create, fromBinary, toBinary } from "@bufbuild/protobuf"; import { FetchRecordsRequestSchema, RecordSchema, @@ -59,7 +58,7 @@ function fetchRecords(client) { const records = []; const call = client.makeServerStreamRequest( FETCH_RECORDS, - (message) => toBinary(FetchRecordsRequestSchema, message), + (message) => Buffer.from(toBinary(FetchRecordsRequestSchema, message)), (bytes) => fromBinary(RecordSchema, bytes), create(FetchRecordsRequestSchema), ); @@ -71,10 +70,10 @@ function fetchRecords(client) { function submitResult(client, result) { return new Promise((resolve, reject) => { - const request = create(SubmitResultRequestSchema, { result: fromJson(StructSchema, result) }); + const request = create(SubmitResultRequestSchema, { result }); client.makeUnaryRequest( SUBMIT_RESULT, - (message) => toBinary(SubmitResultRequestSchema, message), + (message) => Buffer.from(toBinary(SubmitResultRequestSchema, message)), (bytes) => fromBinary(SubmitResultResponseSchema, bytes), request, (error) => (error ? reject(error) : resolve()), @@ -92,7 +91,7 @@ async function main() { try { const records = (await fetchRecords(client)).map((record) => ({ recordId: record.recordId, - data: record.data ? toJson(StructSchema, record.data) : {}, + data: record.data ?? {}, })); console.error(`loaded ${records.length} records over ${socketPath}`); await submitResult(client, analyze(algorithmId, records));