Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ KMS_HOST=kms.sr.secure-rads.org
VERIFIER_HOST=verifier.sr.secure-rads.org
VERIFIER_ATTEST_HOST=verifier-attest.sr.secure-rads.org
VERIFIER_URL=https://verifier.sr.secure-rads.org/algorithms
VERIFIER_RUNS_URL=https://verifier.sr.secure-rads.org/runs

# TLS public-key pins (SHA-256) for the endpoints the enclave and registry-service contact.
# `make env` fills these from the live endpoints.
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
*.pem
!secure/verifier-service/certs/*.pem
secure/csav/registry-service/tls/
secure/sce/execution-service/tls/
secure/csav/registry-service/registry-service
secure/sce/execution-service/execution-service
secure/sce-unit/sce-unit
Expand Down
7 changes: 4 additions & 3 deletions docs/cost-service.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,10 @@ rads (`csav/registry-service`, Lambdas):
- result-consumer (`batchSize` 1, so each run's receive + invoke lands on that run): `sqs:receive` (message
bytes), `sqs:delete` (one request when the handler succeeds; Lambda performs the deletion),
`dynamodb:write-capacity`, `lambda:invocation`, `lambda:compute`.
- verifier-service (secure-rads): `/challenge`, `/attest` and `/refresh` (the enclave's challenge, attestation and
token refresh, keyed by the run) and `/algorithms` (build-time PCR registration, keyed by `algorithmId`): `dynamodb:read-capacity` and
`dynamodb:write-capacity` where attributable, plus `apigateway:request`, `lambda:invocation`, `lambda:compute`.
- verifier-service (secure-rads): `/runs`, `/challenge`, `/attest` and `/refresh` (the run binding, enclave
challenge, attestation and token refresh, keyed by the run) and `/algorithms` (build-time PCR registration,
keyed by `algorithmId`): `dynamodb:read-capacity` and `dynamodb:write-capacity` where attributable, plus
`apigateway:request`, `lambda:invocation`, `lambda:compute`.
- sce-unit (in the enclave) performs no billable AWS operations - its KMS call is metered on kms-service.

## EC2 timing
Expand Down
31 changes: 17 additions & 14 deletions docs/e2e-secure-rads.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,24 @@ Input:
a ← algorithm id
n ← dataset size (# of records)
1 runId ← NEW-ID()
2 emit run-requested
3 cost dynamodb:read-capacity // execution-service: algorithm lookup
4 key ← LOOKUP-EIF(a) // S3 location of algorithm's EIF
5 RUN-COUNT-CHECK(n)
6 cost dynamodb:read-capacity // execution-service Query on the PDM table
7 RunInstances(Nitro parent, userdata ← ⟨a, runId, n, key, launchEpoch⟩)
8 emit executor-launched
9 cost apigateway:request + lambda // per control-plane request/poll (GET /run/{runId}?algorithmId=... too)
10 return runId
2 cost dynamodb:read-capacity // execution-service: algorithm lookup
3 key ← LOOKUP-EIF(a) // S3 location of algorithm's EIF
4 RUN-COUNT-CHECK(n)
5 cost dynamodb:read-capacity // execution-service Query on the PDM table
6 secret ← NEW-SECRET()
7 REGISTER-RUN(Verifier, a, runId, secret) // mTLS; verifier stores SHA-256(secret) ↦ runId, 1h TTL
8 emit run-requested
9 RunInstances(Nitro parent, userdata ← ⟨a, runId, secret, n, key, launchEpoch⟩)
10 emit executor-launched
11 cost apigateway:request + lambda // per control-plane request/poll (GET /run/{runId}?algorithmId=... too)
12 return runId


SECURE-RADS-EXECUTOR(a, runId, n, key)
SECURE-RADS-EXECUTOR(a, runId, secret, n, key)
Input:
a ← algorithm id
runId ← run id
secret ← per-run secret registered with the verifier
n ← dataset size (# of records)
key ← S3 location of algorithm's EIF
1 install nitro-cli
Expand All @@ -50,7 +53,7 @@ Input:
4 cost s3:get // EIF pull
5 EIF ← GET-S3(key)
6 emit artifact-retrieved // direct, auth none
7 start vsock listeners // output :5000, runInfo :5001
7 start vsock listeners // output :5000, runInfo :5001 (runId, secret, size)
8 configure vsock-proxy
9 RUN-ENCLAVE(EIF)
10 emit enclave-launched // direct, auth none
Expand All @@ -66,12 +69,12 @@ SCE-UNIT(a)
Input:
a ← algorithm id
1 cfg ← LOAD-RUN-CONFIG()
2 ⟨runId, n⟩ ← READ-RUNINFO(:5001)
2 ⟨runId, secret, n⟩ ← READ-RUNINFO(:5001)
3 emit enclave-started // no token yet, auth none
4 SEED-ENTROPY from NSM // no AWS cost
5 emit unit-ready // no token yet, auth none
6 challenge ← REQUEST-CHALLENGE(Verifier, a, runId) // single-use nonce, verifier API GW/Lambda and DynamoDB metered
7 token ← ATTEST(Verifier, runId, challenge) // NSM nonce ← challenge; verifier API GW/Lambda and DynamoDB metered
6 challenge ← REQUEST-CHALLENGE(Verifier, a, runId, secret) // rejects a runId whose secret is not registered; single-use nonce, verifier API GW/Lambda and DynamoDB metered
7 token ← ATTEST(Verifier, runId, challenge, SHA-256(secret)) // NSM nonce ← challenge, user_data ← secret hash; verifier API GW/Lambda and DynamoDB metered
8 emit attested
9 records ← FETCH-PDM(a, token, limit ← n)
10 cost apigateway:request + lambda:invocation + lambda:compute + dynamodb:read-capacity // PDM /records
Expand Down
4 changes: 3 additions & 1 deletion docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ Generate the mTLS client certificate, then deploy verifier-service once to creat

```sh
make -C secure/csav/registry-service gen-client-cert
make -C secure/sce gen-client-cert
(cd secure/verifier-service && bun install && bunx serverless deploy \
--param="region=$REGION" \
--param="domainName=verifier.sr.secure-rads.org" \
Expand All @@ -167,8 +168,9 @@ Upload the truststore, then deploy verifier-service again with its immutable S3
TRUSTSTORE_BUCKET=$(aws cloudformation describe-stacks --region "$REGION" \
--stack-name verifier-service-dev \
--query "Stacks[0].Outputs[?OutputKey=='TruststoreBucketName'].OutputValue" --output text)
cat secure/csav/registry-service/tls/client.crt secure/sce/execution-service/tls/client.crt > /tmp/truststore.pem
TRUSTSTORE_VERSION=$(aws s3api put-object --bucket "$TRUSTSTORE_BUCKET" \
--key truststore.pem --body secure/csav/registry-service/tls/client.crt \
--key truststore.pem --body /tmp/truststore.pem \
--query VersionId --output text)
(cd secure/verifier-service && bunx serverless deploy \
--param="region=$REGION" \
Expand Down
4 changes: 4 additions & 0 deletions packages/common/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@
"./jwt": {
"types": "./src/jwt.ts",
"import": "./src/jwt.ts"
},
"./tls": {
"types": "./src/tls.ts",
"import": "./src/tls.ts"
}
},
"scripts": {
Expand Down
7 changes: 4 additions & 3 deletions secure/sce-unit/README.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
# sce-unit

The composed image's real entrypoint. Runs as root, requests a single-use verifier
challenge, attests to the Verifier Service with that challenge as the NSM nonce,
fetches and decrypts
The composed image's real entrypoint. Runs as root, reads this run's id, secret,
and record limit from the host over vsock (`:5001`), requests a single-use verifier
challenge with that id and secret, attests to the Verifier Service with the
challenge as the NSM nonce and SHA-256(secret) as `user_data`, fetches and decrypts
PDM records, writes `records.json` under the DC rootfs `/data` (JSON shape
`algorithmId` + `records`, same load path as rads), starts nsjail with
`RECORDS_PATH` / `RESULT_PATH` set, reads the DC app's `result.json`, then streams
Expand Down
15 changes: 10 additions & 5 deletions secure/sce-unit/attest.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
Expand All @@ -19,15 +20,15 @@ import (

const attestRequestTimeout = 30 * time.Second

func attest(cfg runConfig, runID string) (tokenPair, error) {
func attest(cfg runConfig, runID, runSecret string) (tokenPair, error) {
priv, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
if err != nil {
return tokenPair{}, fmt.Errorf("generate ephemeral key: %w", err)
}
pubKey := elliptic.Marshal(elliptic.P384(), priv.PublicKey.X, priv.PublicKey.Y)

// The verifier issues the nonce, so the document it receives is bound to this specific /attest request.
challenge, err := fetchChallenge(cfg, runID)
challenge, err := fetchChallenge(cfg, runID, runSecret)
if err != nil {
return tokenPair{}, fmt.Errorf("request challenge: %w", err)
}
Expand All @@ -42,7 +43,10 @@ func attest(cfg runConfig, runID string) (tokenPair, error) {
}
defer sess.Close()

res, err := sess.Send(&request.Attestation{Nonce: nonce, PublicKey: pubKey})
// user_data binds the attested session to the run execution-service registered this secret for; /challenge
// already rejected a host that does not hold it.
userData := sha256.Sum256([]byte(runSecret))
res, err := sess.Send(&request.Attestation{Nonce: nonce, PublicKey: pubKey, UserData: userData[:]})
if err != nil {
return tokenPair{}, fmt.Errorf("request attestation document: %w", err)
}
Expand Down Expand Up @@ -73,11 +77,12 @@ func attest(cfg runConfig, runID string) (tokenPair, error) {

// fetchChallenge gets a single-use nonce from verifier-service before the NSM attestation request, so the
// document it signs is valid for one /attest call only.
func fetchChallenge(cfg runConfig, runID string) (string, error) {
func fetchChallenge(cfg runConfig, runID, runSecret string) (string, error) {
body, err := json.Marshal(struct {
AlgorithmId string `json:"algorithmId"`
RunId string `json:"runId"`
}{AlgorithmId: cfg.AlgorithmId, RunId: runID})
RunSecret string `json:"runSecret"`
}{AlgorithmId: cfg.AlgorithmId, RunId: runID, RunSecret: runSecret})
if err != nil {
return "", err
}
Expand Down
12 changes: 9 additions & 3 deletions secure/sce-unit/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func main() {
timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventUnitReady, nil)

// Attest, fetch, decrypt, and write records.json before nsjail starts - same file contract as rads.
subjectIDs, err := prepareRecordsFile(cfg, timelineLog, runID, info.Size)
subjectIDs, err := prepareRecordsFile(cfg, timelineLog, runID, info.RunSecret, info.Size)
if err != nil {
failRun(timelineLog, cfg.AlgorithmId, runID, err)
}
Expand Down Expand Up @@ -139,8 +139,14 @@ func failRun(timelineLog *sharedlog.Client, algorithmID, runID string, err error
// prepareRecordsFile attests, fetches ciphertext from pdm-service, decrypts, and writes the rads-shaped
// records.json the DC app loads via RECORDS_PATH. It also returns the data subjects whose records were
// handed to the DC app, for the post-completion subject-access trail.
func prepareRecordsFile(cfg runConfig, timelineLog *sharedlog.Client, runID string, limit int) ([]string, error) {
tp, err := attest(cfg, runID)
func prepareRecordsFile(
cfg runConfig,
timelineLog *sharedlog.Client,
runID string,
runSecret string,
limit int,
) ([]string, error) {
tp, err := attest(cfg, runID, runSecret)
if err != nil {
return nil, fmt.Errorf("attestation failed: %w", err)
}
Expand Down
14 changes: 8 additions & 6 deletions secure/sce-unit/runinfo.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@ import (
// no other way to learn the per-run id it was launched for (the EIF only carries the per-algorithm config).
const hostRunIDPort = 5001

// runInfo is the per-run data the host serves on hostRunIDPort: the run id and how many records to read
// (0 = the whole dataset). The enclave takes no user-data, so this is its only source for both.
// runInfo is the per-run data the host serves on hostRunIDPort: the run id, the run secret execution-service
// registered for it, and how many records to read (0 = the whole dataset). The host is untrusted, so these
// are claims: the secret only proves itself to verifier-service, never to the enclave.
type runInfo struct {
RunID string `json:"runId"`
Size int `json:"size"`
RunID string `json:"runId"`
RunSecret string `json:"runSecret"`
Size int `json:"size"`
}

func readRunInfo() (runInfo, error) {
Expand Down Expand Up @@ -63,8 +65,8 @@ func readRunInfo() (runInfo, error) {
if err := json.Unmarshal(buf.Bytes(), &info); err != nil {
return runInfo{}, fmt.Errorf("decode run-info: %w", err)
}
if info.RunID == "" || info.Size < 0 {
return runInfo{}, fmt.Errorf("invalid run-info: runId must be nonempty and size must be nonnegative")
if info.RunID == "" || info.RunSecret == "" || info.Size < 0 {
return runInfo{}, fmt.Errorf("invalid run-info: runId and runSecret must be nonempty and size nonnegative")
}
return info, nil
}
15 changes: 13 additions & 2 deletions secure/sce/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ RUN_RESULTS_QUEUE_URL = $(shell aws cloudformation describe-stacks --stack-name
API_URL = $(shell aws cloudformation describe-stacks --stack-name $(STACK_NAME) --region $(REGION) \
--query "Stacks[0].Outputs[?OutputKey=='ExecutionApiUrl'].OutputValue" --output text)

.PHONY: deploy destroy status logs env-check
.PHONY: deploy destroy status logs env-check gen-client-cert

deploy: env-check
deploy: env-check gen-client-cert
cd execution-service && bun install && bunx serverless deploy \
--param="region=$(REGION)" \
--param="vpcId=$(DEFAULT_VPC_ID)" \
Expand All @@ -44,11 +44,22 @@ status:
logs:
aws logs tail /aws/lambda/secure-rads-execution-service-dev-run --region $(REGION) --follow

# Self-signed client cert for mTLS to verifier-service /runs - PoC only, baked into the Lambda package. Its
# public half must be in the verifier truststore (see verifier-service README).
gen-client-cert:
@mkdir -p execution-service/tls
@test -f execution-service/tls/client.key || openssl req -x509 -newkey ec \
-pkeyopt ec_paramgen_curve:prime256v1 -days 3650 -nodes \
-keyout execution-service/tls/client.key -out execution-service/tls/client.crt \
-subj "/CN=execution-service"

env-check:
@$(call env_require,AWS_REGION)
@$(call env_require,AVAILABILITY_ZONE)
@$(call env_require,SCE_ENCLAVE_AMI_ID)
@$(call env_require,VERIFIER_ATTEST_HOST)
@$(call env_require,VERIFIER_RUNS_URL)
@$(call env_require,VERIFIER_PUBKEY_SHA256)
@$(call env_require,PDM_HOST)
@$(call env_require,KMS_HOST)
@$(call env_require,LOG_HOST_SECURE_RADS)
Expand Down
4 changes: 3 additions & 1 deletion secure/sce/execution-service/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ Part of the SCE. API Gateway invokes the TypeScript Lambda control plane. Given

`POST /run` is fire-and-forget: it assigns a `runId`, looks up the EIF, and — when `size` is set — rejects the request if that count exceeds the rows already seeded for the algorithm in PDM. It then launches the enclave and returns `202 {runId, algorithmId, instanceId, status: "pending"}` immediately. The enclave host sends its result (exit status plus what the DC app returned over vsock) to the run-results SQS queue when it finishes; a consumer Lambda (result-consumer) writes it to the RunResults table.

Before launching the instance, `POST /run` registers `(runId, runSecret)` with verifier-service over mTLS (`VERIFIER_RUNS_URL`, pinned by `VERIFIER_PUBKEY_SHA256`, client cert in `tls/`). The secret goes into the host's user data and is served to the enclave on vsock port 5001; the enclave presents it to `/challenge`, so a compromised host cannot bind its enclave to another run's id. Registration happens before `RunInstances`, so a register failure never leaves an unbindable instance running.

`GET /run/{runId}?algorithmId={algorithmId}` polls for the result: `{runId, status: "pending"}` until it lands, then the stored result once the consumer has written it. `algorithmId` is required so every poll can be attributed to its run.

The control plane logs `run-requested` and `executor-launched` (or `launch-failed`). The parent posts `executor-ready`, `artifact-retrieved`, `enclave-launched`, and `executor-finished` directly to telemetry-service with `origin: host`. The host holds no credential; telemetry ingest is unauthenticated. The enclave emits `run-failed` before it exits. result-consumer only emits `result-persisted`. SCE-unit and result-consumer emit the remaining events in [the execution flow](../../../docs/e2e-secure-rads.txt).
Expand All @@ -12,6 +14,6 @@ See [../README.md](../README.md) for the full SCE context.

## Deployment

`make -C sce deploy` deploys the Lambda stack. It requires result-consumer, registry-service storage, and kms-service to exist first.
`make -C sce deploy` deploys the Lambda stack. It requires result-consumer, registry-service storage, kms-service, and verifier-service to exist first. The deploy target generates `tls/client.crt`/`client.key` on first run; add the public cert to the verifier truststore (see verifier-service's README).

The enclave host pulls the EIF and publishes the result; it has no algorithm-key KMS permission. Data-key decryption runs in kms-service, so the per-algorithm key policy names the kms-service role rather than this per-deployment host role. `KMS_HOST` is passed as `KMS_SERVICE_HOST` and baked into the host user data, which proxies the enclave's `/decrypt` call.
4 changes: 4 additions & 0 deletions secure/sce/execution-service/serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ build:
package:
patterns:
- userdata.sh.tmpl
- tls/**

provider:
name: aws
Expand Down Expand Up @@ -55,6 +56,9 @@ provider:
ENCLAVE_SECURITY_GROUP_ID: !Ref EnclaveHostSecurityGroup
ENCLAVE_INSTANCE_PROFILE_ARN: !GetAtt EnclaveHostInstanceProfile.Arn
VERIFIER_HOST: ${env:VERIFIER_ATTEST_HOST}
# mTLS /runs endpoint and the mTLS domain's pin; the client cert/key are packaged from tls/.
VERIFIER_RUNS_URL: ${env:VERIFIER_RUNS_URL}
VERIFIER_PUBKEY_SHA256: ${env:VERIFIER_PUBKEY_SHA256}
PDM_HOST: ${env:PDM_HOST}
KMS_SERVICE_HOST: ${env:KMS_HOST}
PDM_TABLE: ${param:pdmTable}
Expand Down
6 changes: 6 additions & 0 deletions secure/sce/execution-service/src/handlers/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { requiredEnv } from "@srads/common/env";
import { emitEvent } from "@srads/common/log";
import type { APIGatewayProxyHandlerV2 } from "aws-lambda";
import { countAlgorithmRecords, isThrottlingError } from "../lib/pdmCount.js";
import { newRunSecret, registerRunBinding } from "../lib/runBinding.js";
import { userData } from "../lib/userdata.js";

const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({}));
Expand Down Expand Up @@ -134,6 +135,10 @@ export const create: APIGatewayProxyHandlerV2 = async (event) => {
}
}

// Registers the binding before launch so a failed register never leaves an instance running unbindable.
const runSecret = newRunSecret();
await registerRunBinding(request.algorithmId, runId, runSecret);

await emitEvent(request.algorithmId, runId, "run-requested");
const launched = await ec2.send(
new RunInstancesCommand({
Expand All @@ -157,6 +162,7 @@ export const create: APIGatewayProxyHandlerV2 = async (event) => {
region,
queueUrl: runResultsQueueUrl,
runId,
runSecret,
s3Bucket: record.S3Bucket,
s3Key: record.S3Key,
size: request.size,
Expand Down
Loading