diff --git a/.env.example b/.env.example index 36c1f43..7b4f114 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,7 @@ KMS_HOST=kms.sr.secure-rads.org VERIFIER_HOST=verifier.sr.secure-rads.org VERIFIER_ATTEST_HOST=verifier-attest.sr.secure-rads.org VERIFIER_URL=https://verifier.sr.secure-rads.org/algorithms +VERIFIER_RUNS_URL=https://verifier.sr.secure-rads.org/runs # TLS public-key pins (SHA-256) for the endpoints the enclave and registry-service contact. # `make env` fills these from the live endpoints. diff --git a/.gitignore b/.gitignore index 7525f39..5278b82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pem !secure/verifier-service/certs/*.pem secure/csav/registry-service/tls/ +secure/sce/execution-service/tls/ secure/csav/registry-service/registry-service secure/sce/execution-service/execution-service secure/sce-unit/sce-unit diff --git a/docs/cost-service.md b/docs/cost-service.md index 166b302..72f9c51 100644 --- a/docs/cost-service.md +++ b/docs/cost-service.md @@ -63,9 +63,10 @@ rads (`csav/registry-service`, Lambdas): - result-consumer (`batchSize` 1, so each run's receive + invoke lands on that run): `sqs:receive` (message bytes), `sqs:delete` (one request when the handler succeeds; Lambda performs the deletion), `dynamodb:write-capacity`, `lambda:invocation`, `lambda:compute`. -- verifier-service (secure-rads): `/challenge`, `/attest` and `/refresh` (the enclave's challenge, attestation and - token refresh, keyed by the run) and `/algorithms` (build-time PCR registration, keyed by `algorithmId`): `dynamodb:read-capacity` and - `dynamodb:write-capacity` where attributable, plus `apigateway:request`, `lambda:invocation`, `lambda:compute`. +- verifier-service (secure-rads): `/runs`, `/challenge`, `/attest` and `/refresh` (the run binding, enclave + challenge, attestation and token refresh, keyed by the run) and `/algorithms` (build-time PCR registration, + keyed by `algorithmId`): `dynamodb:read-capacity` and `dynamodb:write-capacity` where attributable, plus + `apigateway:request`, `lambda:invocation`, `lambda:compute`. - sce-unit (in the enclave) performs no billable AWS operations - its KMS call is metered on kms-service. ## EC2 timing diff --git a/docs/e2e-secure-rads.txt b/docs/e2e-secure-rads.txt index d88fe76..b691ff9 100644 --- a/docs/e2e-secure-rads.txt +++ b/docs/e2e-secure-rads.txt @@ -27,21 +27,24 @@ Input: a ← algorithm id n ← dataset size (# of records) 1 runId ← NEW-ID() -2 emit run-requested -3 cost dynamodb:read-capacity // execution-service: algorithm lookup -4 key ← LOOKUP-EIF(a) // S3 location of algorithm's EIF -5 RUN-COUNT-CHECK(n) -6 cost dynamodb:read-capacity // execution-service Query on the PDM table -7 RunInstances(Nitro parent, userdata ← ⟨a, runId, n, key, launchEpoch⟩) -8 emit executor-launched -9 cost apigateway:request + lambda // per control-plane request/poll (GET /run/{runId}?algorithmId=... too) -10 return runId +2 cost dynamodb:read-capacity // execution-service: algorithm lookup +3 key ← LOOKUP-EIF(a) // S3 location of algorithm's EIF +4 RUN-COUNT-CHECK(n) +5 cost dynamodb:read-capacity // execution-service Query on the PDM table +6 secret ← NEW-SECRET() +7 REGISTER-RUN(Verifier, a, runId, secret) // mTLS; verifier stores SHA-256(secret) ↦ runId, 1h TTL +8 emit run-requested +9 RunInstances(Nitro parent, userdata ← ⟨a, runId, secret, n, key, launchEpoch⟩) +10 emit executor-launched +11 cost apigateway:request + lambda // per control-plane request/poll (GET /run/{runId}?algorithmId=... too) +12 return runId -SECURE-RADS-EXECUTOR(a, runId, n, key) +SECURE-RADS-EXECUTOR(a, runId, secret, n, key) Input: a ← algorithm id runId ← run id + secret ← per-run secret registered with the verifier n ← dataset size (# of records) key ← S3 location of algorithm's EIF 1 install nitro-cli @@ -50,7 +53,7 @@ Input: 4 cost s3:get // EIF pull 5 EIF ← GET-S3(key) 6 emit artifact-retrieved // direct, auth none -7 start vsock listeners // output :5000, runInfo :5001 +7 start vsock listeners // output :5000, runInfo :5001 (runId, secret, size) 8 configure vsock-proxy 9 RUN-ENCLAVE(EIF) 10 emit enclave-launched // direct, auth none @@ -66,12 +69,12 @@ SCE-UNIT(a) Input: a ← algorithm id 1 cfg ← LOAD-RUN-CONFIG() -2 ⟨runId, n⟩ ← READ-RUNINFO(:5001) +2 ⟨runId, secret, n⟩ ← READ-RUNINFO(:5001) 3 emit enclave-started // no token yet, auth none 4 SEED-ENTROPY from NSM // no AWS cost 5 emit unit-ready // no token yet, auth none -6 challenge ← REQUEST-CHALLENGE(Verifier, a, runId) // single-use nonce, verifier API GW/Lambda and DynamoDB metered -7 token ← ATTEST(Verifier, runId, challenge) // NSM nonce ← challenge; verifier API GW/Lambda and DynamoDB metered +6 challenge ← REQUEST-CHALLENGE(Verifier, a, runId, secret) // rejects a runId whose secret is not registered; single-use nonce, verifier API GW/Lambda and DynamoDB metered +7 token ← ATTEST(Verifier, runId, challenge, SHA-256(secret)) // NSM nonce ← challenge, user_data ← secret hash; verifier API GW/Lambda and DynamoDB metered 8 emit attested 9 records ← FETCH-PDM(a, token, limit ← n) 10 cost apigateway:request + lambda:invocation + lambda:compute + dynamodb:read-capacity // PDM /records diff --git a/docs/getting-started.md b/docs/getting-started.md index ac3c06e..6572657 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -153,6 +153,7 @@ Generate the mTLS client certificate, then deploy verifier-service once to creat ```sh make -C secure/csav/registry-service gen-client-cert +make -C secure/sce gen-client-cert (cd secure/verifier-service && bun install && bunx serverless deploy \ --param="region=$REGION" \ --param="domainName=verifier.sr.secure-rads.org" \ @@ -167,8 +168,9 @@ Upload the truststore, then deploy verifier-service again with its immutable S3 TRUSTSTORE_BUCKET=$(aws cloudformation describe-stacks --region "$REGION" \ --stack-name verifier-service-dev \ --query "Stacks[0].Outputs[?OutputKey=='TruststoreBucketName'].OutputValue" --output text) +cat secure/csav/registry-service/tls/client.crt secure/sce/execution-service/tls/client.crt > /tmp/truststore.pem TRUSTSTORE_VERSION=$(aws s3api put-object --bucket "$TRUSTSTORE_BUCKET" \ - --key truststore.pem --body secure/csav/registry-service/tls/client.crt \ + --key truststore.pem --body /tmp/truststore.pem \ --query VersionId --output text) (cd secure/verifier-service && bunx serverless deploy \ --param="region=$REGION" \ diff --git a/packages/common/package.json b/packages/common/package.json index 556e362..2844507 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -19,6 +19,10 @@ "./jwt": { "types": "./src/jwt.ts", "import": "./src/jwt.ts" + }, + "./tls": { + "types": "./src/tls.ts", + "import": "./src/tls.ts" } }, "scripts": { diff --git a/secure/sce-unit/README.md b/secure/sce-unit/README.md index a6245c8..a3e9d27 100644 --- a/secure/sce-unit/README.md +++ b/secure/sce-unit/README.md @@ -1,8 +1,9 @@ # sce-unit -The composed image's real entrypoint. Runs as root, requests a single-use verifier -challenge, attests to the Verifier Service with that challenge as the NSM nonce, -fetches and decrypts +The composed image's real entrypoint. Runs as root, reads this run's id, secret, +and record limit from the host over vsock (`:5001`), requests a single-use verifier +challenge with that id and secret, attests to the Verifier Service with the +challenge as the NSM nonce and SHA-256(secret) as `user_data`, fetches and decrypts PDM records, writes `records.json` under the DC rootfs `/data` (JSON shape `algorithmId` + `records`, same load path as rads), starts nsjail with `RECORDS_PATH` / `RESULT_PATH` set, reads the DC app's `result.json`, then streams diff --git a/secure/sce-unit/attest.go b/secure/sce-unit/attest.go index 6e6235e..fc7b388 100644 --- a/secure/sce-unit/attest.go +++ b/secure/sce-unit/attest.go @@ -6,6 +6,7 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/rand" + "crypto/sha256" "encoding/base64" "encoding/json" "fmt" @@ -19,7 +20,7 @@ import ( const attestRequestTimeout = 30 * time.Second -func attest(cfg runConfig, runID string) (tokenPair, error) { +func attest(cfg runConfig, runID, runSecret string) (tokenPair, error) { priv, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) if err != nil { return tokenPair{}, fmt.Errorf("generate ephemeral key: %w", err) @@ -27,7 +28,7 @@ func attest(cfg runConfig, runID string) (tokenPair, error) { pubKey := elliptic.Marshal(elliptic.P384(), priv.PublicKey.X, priv.PublicKey.Y) // The verifier issues the nonce, so the document it receives is bound to this specific /attest request. - challenge, err := fetchChallenge(cfg, runID) + challenge, err := fetchChallenge(cfg, runID, runSecret) if err != nil { return tokenPair{}, fmt.Errorf("request challenge: %w", err) } @@ -42,7 +43,10 @@ func attest(cfg runConfig, runID string) (tokenPair, error) { } defer sess.Close() - res, err := sess.Send(&request.Attestation{Nonce: nonce, PublicKey: pubKey}) + // user_data binds the attested session to the run execution-service registered this secret for; /challenge + // already rejected a host that does not hold it. + userData := sha256.Sum256([]byte(runSecret)) + res, err := sess.Send(&request.Attestation{Nonce: nonce, PublicKey: pubKey, UserData: userData[:]}) if err != nil { return tokenPair{}, fmt.Errorf("request attestation document: %w", err) } @@ -73,11 +77,12 @@ func attest(cfg runConfig, runID string) (tokenPair, error) { // fetchChallenge gets a single-use nonce from verifier-service before the NSM attestation request, so the // document it signs is valid for one /attest call only. -func fetchChallenge(cfg runConfig, runID string) (string, error) { +func fetchChallenge(cfg runConfig, runID, runSecret string) (string, error) { body, err := json.Marshal(struct { AlgorithmId string `json:"algorithmId"` RunId string `json:"runId"` - }{AlgorithmId: cfg.AlgorithmId, RunId: runID}) + RunSecret string `json:"runSecret"` + }{AlgorithmId: cfg.AlgorithmId, RunId: runID, RunSecret: runSecret}) if err != nil { return "", err } diff --git a/secure/sce-unit/main.go b/secure/sce-unit/main.go index 1dd91e1..f214d99 100644 --- a/secure/sce-unit/main.go +++ b/secure/sce-unit/main.go @@ -64,7 +64,7 @@ func main() { timelineLog.EmitEvent(cfg.AlgorithmId, &runID, sharedlog.EventUnitReady, nil) // Attest, fetch, decrypt, and write records.json before nsjail starts - same file contract as rads. - subjectIDs, err := prepareRecordsFile(cfg, timelineLog, runID, info.Size) + subjectIDs, err := prepareRecordsFile(cfg, timelineLog, runID, info.RunSecret, info.Size) if err != nil { failRun(timelineLog, cfg.AlgorithmId, runID, err) } @@ -139,8 +139,14 @@ func failRun(timelineLog *sharedlog.Client, algorithmID, runID string, err error // prepareRecordsFile attests, fetches ciphertext from pdm-service, decrypts, and writes the rads-shaped // records.json the DC app loads via RECORDS_PATH. It also returns the data subjects whose records were // handed to the DC app, for the post-completion subject-access trail. -func prepareRecordsFile(cfg runConfig, timelineLog *sharedlog.Client, runID string, limit int) ([]string, error) { - tp, err := attest(cfg, runID) +func prepareRecordsFile( + cfg runConfig, + timelineLog *sharedlog.Client, + runID string, + runSecret string, + limit int, +) ([]string, error) { + tp, err := attest(cfg, runID, runSecret) if err != nil { return nil, fmt.Errorf("attestation failed: %w", err) } diff --git a/secure/sce-unit/runinfo.go b/secure/sce-unit/runinfo.go index 5013b7d..0cd8556 100644 --- a/secure/sce-unit/runinfo.go +++ b/secure/sce-unit/runinfo.go @@ -14,11 +14,13 @@ import ( // no other way to learn the per-run id it was launched for (the EIF only carries the per-algorithm config). const hostRunIDPort = 5001 -// runInfo is the per-run data the host serves on hostRunIDPort: the run id and how many records to read -// (0 = the whole dataset). The enclave takes no user-data, so this is its only source for both. +// runInfo is the per-run data the host serves on hostRunIDPort: the run id, the run secret execution-service +// registered for it, and how many records to read (0 = the whole dataset). The host is untrusted, so these +// are claims: the secret only proves itself to verifier-service, never to the enclave. type runInfo struct { - RunID string `json:"runId"` - Size int `json:"size"` + RunID string `json:"runId"` + RunSecret string `json:"runSecret"` + Size int `json:"size"` } func readRunInfo() (runInfo, error) { @@ -63,8 +65,8 @@ func readRunInfo() (runInfo, error) { if err := json.Unmarshal(buf.Bytes(), &info); err != nil { return runInfo{}, fmt.Errorf("decode run-info: %w", err) } - if info.RunID == "" || info.Size < 0 { - return runInfo{}, fmt.Errorf("invalid run-info: runId must be nonempty and size must be nonnegative") + if info.RunID == "" || info.RunSecret == "" || info.Size < 0 { + return runInfo{}, fmt.Errorf("invalid run-info: runId and runSecret must be nonempty and size nonnegative") } return info, nil } diff --git a/secure/sce/Makefile b/secure/sce/Makefile index 6ae2df1..dd08056 100644 --- a/secure/sce/Makefile +++ b/secure/sce/Makefile @@ -21,9 +21,9 @@ RUN_RESULTS_QUEUE_URL = $(shell aws cloudformation describe-stacks --stack-name API_URL = $(shell aws cloudformation describe-stacks --stack-name $(STACK_NAME) --region $(REGION) \ --query "Stacks[0].Outputs[?OutputKey=='ExecutionApiUrl'].OutputValue" --output text) -.PHONY: deploy destroy status logs env-check +.PHONY: deploy destroy status logs env-check gen-client-cert -deploy: env-check +deploy: env-check gen-client-cert cd execution-service && bun install && bunx serverless deploy \ --param="region=$(REGION)" \ --param="vpcId=$(DEFAULT_VPC_ID)" \ @@ -44,11 +44,22 @@ status: logs: aws logs tail /aws/lambda/secure-rads-execution-service-dev-run --region $(REGION) --follow +# Self-signed client cert for mTLS to verifier-service /runs - PoC only, baked into the Lambda package. Its +# public half must be in the verifier truststore (see verifier-service README). +gen-client-cert: + @mkdir -p execution-service/tls + @test -f execution-service/tls/client.key || openssl req -x509 -newkey ec \ + -pkeyopt ec_paramgen_curve:prime256v1 -days 3650 -nodes \ + -keyout execution-service/tls/client.key -out execution-service/tls/client.crt \ + -subj "/CN=execution-service" + env-check: @$(call env_require,AWS_REGION) @$(call env_require,AVAILABILITY_ZONE) @$(call env_require,SCE_ENCLAVE_AMI_ID) @$(call env_require,VERIFIER_ATTEST_HOST) + @$(call env_require,VERIFIER_RUNS_URL) + @$(call env_require,VERIFIER_PUBKEY_SHA256) @$(call env_require,PDM_HOST) @$(call env_require,KMS_HOST) @$(call env_require,LOG_HOST_SECURE_RADS) diff --git a/secure/sce/execution-service/README.md b/secure/sce/execution-service/README.md index df4c6ad..aa1d00a 100644 --- a/secure/sce/execution-service/README.md +++ b/secure/sce/execution-service/README.md @@ -4,6 +4,8 @@ Part of the SCE. API Gateway invokes the TypeScript Lambda control plane. Given `POST /run` is fire-and-forget: it assigns a `runId`, looks up the EIF, and — when `size` is set — rejects the request if that count exceeds the rows already seeded for the algorithm in PDM. It then launches the enclave and returns `202 {runId, algorithmId, instanceId, status: "pending"}` immediately. The enclave host sends its result (exit status plus what the DC app returned over vsock) to the run-results SQS queue when it finishes; a consumer Lambda (result-consumer) writes it to the RunResults table. +Before launching the instance, `POST /run` registers `(runId, runSecret)` with verifier-service over mTLS (`VERIFIER_RUNS_URL`, pinned by `VERIFIER_PUBKEY_SHA256`, client cert in `tls/`). The secret goes into the host's user data and is served to the enclave on vsock port 5001; the enclave presents it to `/challenge`, so a compromised host cannot bind its enclave to another run's id. Registration happens before `RunInstances`, so a register failure never leaves an unbindable instance running. + `GET /run/{runId}?algorithmId={algorithmId}` polls for the result: `{runId, status: "pending"}` until it lands, then the stored result once the consumer has written it. `algorithmId` is required so every poll can be attributed to its run. The control plane logs `run-requested` and `executor-launched` (or `launch-failed`). The parent posts `executor-ready`, `artifact-retrieved`, `enclave-launched`, and `executor-finished` directly to telemetry-service with `origin: host`. The host holds no credential; telemetry ingest is unauthenticated. The enclave emits `run-failed` before it exits. result-consumer only emits `result-persisted`. SCE-unit and result-consumer emit the remaining events in [the execution flow](../../../docs/e2e-secure-rads.txt). @@ -12,6 +14,6 @@ See [../README.md](../README.md) for the full SCE context. ## Deployment -`make -C sce deploy` deploys the Lambda stack. It requires result-consumer, registry-service storage, and kms-service to exist first. +`make -C sce deploy` deploys the Lambda stack. It requires result-consumer, registry-service storage, kms-service, and verifier-service to exist first. The deploy target generates `tls/client.crt`/`client.key` on first run; add the public cert to the verifier truststore (see verifier-service's README). The enclave host pulls the EIF and publishes the result; it has no algorithm-key KMS permission. Data-key decryption runs in kms-service, so the per-algorithm key policy names the kms-service role rather than this per-deployment host role. `KMS_HOST` is passed as `KMS_SERVICE_HOST` and baked into the host user data, which proxies the enclave's `/decrypt` call. diff --git a/secure/sce/execution-service/serverless.yml b/secure/sce/execution-service/serverless.yml index 87c4f25..50e45be 100644 --- a/secure/sce/execution-service/serverless.yml +++ b/secure/sce/execution-service/serverless.yml @@ -9,6 +9,7 @@ build: package: patterns: - userdata.sh.tmpl + - tls/** provider: name: aws @@ -55,6 +56,9 @@ provider: ENCLAVE_SECURITY_GROUP_ID: !Ref EnclaveHostSecurityGroup ENCLAVE_INSTANCE_PROFILE_ARN: !GetAtt EnclaveHostInstanceProfile.Arn VERIFIER_HOST: ${env:VERIFIER_ATTEST_HOST} + # mTLS /runs endpoint and the mTLS domain's pin; the client cert/key are packaged from tls/. + VERIFIER_RUNS_URL: ${env:VERIFIER_RUNS_URL} + VERIFIER_PUBKEY_SHA256: ${env:VERIFIER_PUBKEY_SHA256} PDM_HOST: ${env:PDM_HOST} KMS_SERVICE_HOST: ${env:KMS_HOST} PDM_TABLE: ${param:pdmTable} diff --git a/secure/sce/execution-service/src/handlers/run.ts b/secure/sce/execution-service/src/handlers/run.ts index 9adab82..f6872fe 100644 --- a/secure/sce/execution-service/src/handlers/run.ts +++ b/secure/sce/execution-service/src/handlers/run.ts @@ -7,6 +7,7 @@ import { requiredEnv } from "@srads/common/env"; import { emitEvent } from "@srads/common/log"; import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { countAlgorithmRecords, isThrottlingError } from "../lib/pdmCount.js"; +import { newRunSecret, registerRunBinding } from "../lib/runBinding.js"; import { userData } from "../lib/userdata.js"; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); @@ -134,6 +135,10 @@ export const create: APIGatewayProxyHandlerV2 = async (event) => { } } + // Registers the binding before launch so a failed register never leaves an instance running unbindable. + const runSecret = newRunSecret(); + await registerRunBinding(request.algorithmId, runId, runSecret); + await emitEvent(request.algorithmId, runId, "run-requested"); const launched = await ec2.send( new RunInstancesCommand({ @@ -157,6 +162,7 @@ export const create: APIGatewayProxyHandlerV2 = async (event) => { region, queueUrl: runResultsQueueUrl, runId, + runSecret, s3Bucket: record.S3Bucket, s3Key: record.S3Key, size: request.size, diff --git a/secure/sce/execution-service/src/lib/runBinding.ts b/secure/sce/execution-service/src/lib/runBinding.ts new file mode 100644 index 0000000..eec4551 --- /dev/null +++ b/secure/sce/execution-service/src/lib/runBinding.ts @@ -0,0 +1,50 @@ +import { randomBytes } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { request } from "node:https"; +import { join } from "node:path"; +import { requiredEnv } from "@srads/common/env"; +import { pinnedServerIdentity } from "@srads/common/tls"; + +const verifierRunsUrl = requiredEnv("VERIFIER_RUNS_URL"); +const verifierPubkeySha256 = requiredEnv("VERIFIER_PUBKEY_SHA256"); +const assetsDir = process.env.LAMBDA_TASK_ROOT ?? process.cwd(); +const REGISTER_TIMEOUT_MS = 10_000; + +export function newRunSecret(): string { + return randomBytes(32).toString("base64url"); +} + +// Registers the (runId, runSecret) pair before the instance launches. The host only ever learns its own run's +// secret, so it cannot obtain a verifier challenge for another run's id. +export async function registerRunBinding(algorithmId: string, runId: string, runSecret: string): Promise { + const body = JSON.stringify({ algorithmId, runId, runSecret }); + const response = await post(body); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw new Error(`verifier-service /runs returned ${response.statusCode}: ${response.body}`); + } +} + +function post(body: string): Promise<{ statusCode: number; body: string }> { + return new Promise((resolve, reject) => { + const req = request( + verifierRunsUrl, + { + method: "POST", + cert: readFileSync(join(assetsDir, "tls", "client.crt")), + key: readFileSync(join(assetsDir, "tls", "client.key")), + checkServerIdentity: pinnedServerIdentity(verifierPubkeySha256, "verifier mTLS server"), + headers: { "content-type": "application/json", "content-length": Buffer.byteLength(body) }, + timeout: REGISTER_TIMEOUT_MS, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (chunk: Buffer) => chunks.push(chunk)); + res.on("end", () => resolve({ statusCode: res.statusCode ?? 0, body: Buffer.concat(chunks).toString("utf8") })); + }, + ); + req.on("timeout", () => req.destroy(new Error("verifier-service /runs timed out"))); + req.on("error", reject); + req.write(body); + req.end(); + }); +} diff --git a/secure/sce/execution-service/src/lib/userdata.ts b/secure/sce/execution-service/src/lib/userdata.ts index 6352602..d0c652e 100644 --- a/secure/sce/execution-service/src/lib/userdata.ts +++ b/secure/sce/execution-service/src/lib/userdata.ts @@ -13,6 +13,7 @@ export interface UserDataValues { pdmHost: string; region: string; runId: string; + runSecret: string; s3Bucket: string; s3Key: string; verifierHost: string; @@ -34,6 +35,7 @@ export function userData(values: UserDataValues): string { QueueUrl: values.queueUrl, Region: values.region, RunId: values.runId, + RunSecret: values.runSecret, S3Bucket: values.s3Bucket, S3Key: values.s3Key, Size: (values.size ?? 0).toString(), diff --git a/secure/sce/execution-service/userdata.sh.tmpl b/secure/sce/execution-service/userdata.sh.tmpl index c56b3a9..6977b85 100644 --- a/secure/sce/execution-service/userdata.sh.tmpl +++ b/secure/sce/execution-service/userdata.sh.tmpl @@ -81,15 +81,16 @@ with open("/tmp/output.txt", "wb") as f: PYEOF OUTPUT_LISTENER_PID=$! -# Serve this run's id and record limit to the enclave on port 5001 - the enclave takes no user-data, so it -# has no other way to learn the per-run id or how many records to read (the EIF carries per-algorithm config). +# Serve this run's id, secret, and record limit to the enclave on port 5001 - the enclave takes no user-data, +# so it has no other way to learn them (the EIF carries per-algorithm config). The enclave treats these as +# untrusted claims and proves the secret to verifier-service before it is issued a token. python3 - <<'PYEOF' & import json, socket s = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((socket.VMADDR_CID_ANY, 5001)) s.listen(5) -body = json.dumps({"runId": "{{.RunId}}", "size": {{.Size}}}).encode() +body = json.dumps({"runId": "{{.RunId}}", "runSecret": "{{.RunSecret}}", "size": {{.Size}}}).encode() while True: conn, _ = s.accept() conn.sendall(body) diff --git a/secure/verifier-service/README.md b/secure/verifier-service/README.md index b66baf2..cf11560 100644 --- a/secure/verifier-service/README.md +++ b/secure/verifier-service/README.md @@ -5,8 +5,9 @@ Serverless Framework v4, TypeScript. Receives `AlgorithmId` + PCR values from re ## Endpoints (two separate custom domains, see below) - `POST /algorithms` (mTLS domain) - registry-service registers `AlgorithmId` -> PCR0/1/2 at build time. -- `POST /challenge` (public domain) - sce-unit asks for a single-use nonce before attesting. The row is stored hashed in `AttestationChallengeTable` with a 60 s TTL and is bound to the `algorithmId` and `runId` in the request. -- `POST /attest` (public domain) - sce-unit submits a real Nitro attestation document (base64 COSE_Sign1 bytes) + `algorithmId` + `runId` + the `challenge` it was issued. The document's `nonce` must equal the challenge, and the challenge is consumed atomically before the PCR check, so a captured document cannot be replayed. Verified against the pinned AWS Nitro root cert chain and the registered PCRs; on success, issues a signed access JWT + opaque refresh token (persisted in `AttestationSessionTable`, including `RunId`). +- `POST /runs` (mTLS domain) - execution-service registers `runId` + a per-run `runSecret` before launching the host. The table stores only SHA-256(`runSecret`) with a 1 h TTL. This is what makes the `runId` a capability instead of a claim the untrusted host can retarget. +- `POST /challenge` (public domain) - sce-unit asks for a single-use nonce before attesting, presenting `algorithmId`, `runId`, and `runSecret`. The request is rejected unless the `(runId, runSecret)` pair matches the registered binding. The challenge row is stored hashed in `AttestationChallengeTable` with a 60 s TTL, bound to `algorithmId`/`runId`, and carries the binding's secret hash forward. +- `POST /attest` (public domain) - sce-unit submits a real Nitro attestation document (base64 COSE_Sign1 bytes) + `algorithmId` + `runId` + the `challenge` it was issued. The document's `nonce` must equal the challenge, and the challenge is consumed atomically before the PCR check, so a captured document cannot be replayed. The document's `user_data` must equal the challenge row's secret hash (sce-unit sets it to SHA-256(`runSecret`)), binding the attested session to the run it was actually launched for. Verified against the pinned AWS Nitro root cert chain and the registered PCRs; on success, issues a signed access JWT + opaque refresh token (persisted in `AttestationSessionTable`, including `RunId`). - `POST /refresh` (public domain) - exchanges a still-valid refresh token plus its `runId` for a new access JWT, without requiring a fresh attestation document. The `runId` must match the one stored on the session. - `GET /.well-known/jwks.json` (public domain) - the ML-DSA-87 public key downstream services use to verify access JWTs locally. Public by definition, no auth. @@ -35,7 +36,7 @@ Bun is used as the local dev/bundle tool only (`bun install`) - the Lambda itsel ## Prerequisites (manual, one-time, outside this repo's IaC) - Two domain names + ACM certificates already provisioned, one per custom domain - `serverless.yml` takes these as `domainName`/`certificateArn` (mTLS) and `publicDomainName`/`publicCertificateArn` (plain TLS) params, it does not create them. -- The truststore PEM in `TruststoreBucket` bundles every mTLS client cert: registry-service's (`secure/csav/registry-service/tls/client.crt`, from `make gen-client-cert` there) and pdm-service's (see `secure/pdm-service/README.md`). `TruststoreBucket` is versioned per AWS's guidance for safe truststore rollovers. +- The truststore PEM in `TruststoreBucket` bundles every mTLS client cert: registry-service's (`secure/csav/registry-service/tls/client.crt`, from `make gen-client-cert` there), execution-service's (`secure/sce/execution-service/tls/client.crt`, from `make -C secure/sce gen-client-cert`), and pdm-service's (see `secure/pdm-service/README.md`). Concatenate all client certs into `truststore.pem` before upload. `TruststoreBucket` is versioned per AWS's guidance for safe truststore rollovers. - Once deployed, each custom domain's TLS leaf cert pubkey SHA-256 needs to be read back and pinned on the other side: the mTLS domain's into registry-service's `VERIFIER_PUBKEY_SHA256` (`secure/csav/registry-service/verifier.go`), the public domain's into registry-service's `VERIFIER_HOST`/pubkey used to bake `sce-unit`'s run config (`secure/csav/registry-service/rootfs.go`), into execution-service's `VERIFIER_HOST` (`secure/sce/execution-service/Makefile`), and into pdm-service's `verifierJwksPubkeySha256` (pinned on JWKS fetches). Manual bootstrap step since both sides pin each other. - An ML-DSA-87 (FIPS 204) JWT signing key, stored as an SSM SecureString at `/verifier-service/jwt-signing-key`: @@ -54,8 +55,9 @@ First deploy the functions, tables, and truststore bucket without `truststoreVer bun install bunx serverless deploy --param="domainName=..." --param="certificateArn=..." --param="publicDomainName=..." --param="publicCertificateArn=..." +cat ../csav/registry-service/tls/client.crt ../sce/execution-service/tls/client.crt > /tmp/truststore.pem aws s3api put-object --bucket --key truststore.pem \ - --body ../csav/registry-service/tls/client.crt --query VersionId --output text + --body /tmp/truststore.pem --query VersionId --output text bunx serverless deploy --param="domainName=..." --param="certificateArn=..." \ --param="publicDomainName=..." --param="publicCertificateArn=..." \ diff --git a/secure/verifier-service/serverless.yml b/secure/verifier-service/serverless.yml index d599a35..8173785 100644 --- a/secure/verifier-service/serverless.yml +++ b/secure/verifier-service/serverless.yml @@ -37,6 +37,11 @@ provider: - dynamodb:PutItem - dynamodb:UpdateItem Resource: !GetAtt AttestationChallengeTable.Arn + - Effect: Allow + Action: + - dynamodb:GetItem + - dynamodb:PutItem + Resource: !GetAtt RunBindingTable.Arn - Effect: Allow Action: ssm:GetParameter Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/secure-rads/verifier-service/jwt-signing-key" @@ -44,6 +49,7 @@ provider: ALGORITHM_PCR_TABLE: !Ref AlgorithmPCRTable ATTESTATION_SESSION_TABLE: !Ref AttestationSessionTable ATTESTATION_CHALLENGE_TABLE: !Ref AttestationChallengeTable + RUN_BINDING_TABLE: !Ref RunBindingTable JWT_SIGNING_KEY_PARAM: /secure-rads/verifier-service/jwt-signing-key COST_HOST: ${env:COST_HOST_SECURE_RADS} @@ -55,6 +61,16 @@ functions: method: POST path: /algorithms + # execution-service registers a per-run secret over the same mTLS domain before launching the host. The host + # relays the secret to the enclave, which presents it to /challenge: without it the host cannot bind an + # attested session to another run's id. + registerRun: + handler: src/handlers/registerRun.handler + events: + - httpApi: + method: POST + path: /runs + # attest/challenge/refresh are plain TLS: wired by hand to PublicHttpApi in resources below, not the mTLS httpApi. challenge: handler: src/handlers/challenge.handler @@ -128,8 +144,25 @@ resources: AttributeName: ExpiresAt Enabled: true - # PEM bundle containing registry-service's client cert public cert - # (csav/registry-service/tls/client.crt) - upload it here before deploy. + # One row per launched run: the hash of the secret execution-service now binds to this RunId. The row is + # what lets /challenge reject a host that tries to run the enclave under another run's id, and what /attest + # compares the document's user_data against. + RunBindingTable: + Type: AWS::DynamoDB::Table + Properties: + BillingMode: PAY_PER_REQUEST + AttributeDefinitions: + - AttributeName: RunId + AttributeType: S + KeySchema: + - AttributeName: RunId + KeyType: HASH + TimeToLiveSpecification: + AttributeName: ExpiresAt + Enabled: true + + # PEM bundle of every mTLS client cert: registry-service's and execution-service's + # (csav/registry-service/tls/client.crt, sce/execution-service/tls/client.crt) - upload it before deploy. TruststoreBucket: Type: AWS::S3::Bucket Properties: diff --git a/secure/verifier-service/src/handlers/challenge.ts b/secure/verifier-service/src/handlers/challenge.ts index cf9ed5b..52d3097 100644 --- a/secure/verifier-service/src/handlers/challenge.ts +++ b/secure/verifier-service/src/handlers/challenge.ts @@ -4,6 +4,7 @@ import { CostPhase, costEvent, recordRequest } from "@srads/common/cost"; import { requiredEnv } from "@srads/common/env"; import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { issueChallenge } from "../lib/challenge.js"; +import { verifyRunBinding } from "../lib/runBinding.js"; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); const algorithmPcrTable = requiredEnv("ALGORITHM_PCR_TABLE"); @@ -11,12 +12,13 @@ const algorithmPcrTable = requiredEnv("ALGORITHM_PCR_TABLE"); interface ChallengeBody { algorithmId: string; runId: string; + runSecret: string; } function isChallengeBody(body: unknown): body is ChallengeBody { if (typeof body !== "object" || body === null) return false; const b = body as Record; - return typeof b.algorithmId === "string" && typeof b.runId === "string"; + return [b.algorithmId, b.runId, b.runSecret].every((value) => typeof value === "string" && value.length > 0); } export const handler: APIGatewayProxyHandlerV2 = async (event) => { @@ -33,7 +35,7 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { } if (!isChallengeBody(body)) { - return { statusCode: 400, body: "expected algorithmId and runId as strings" }; + return { statusCode: 400, body: "expected algorithmId, runId, and runSecret as non-empty strings" }; } try { @@ -57,7 +59,12 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { return { statusCode: 404, body: "algorithmId not registered" }; } - const challenge = await issueChallenge(body.algorithmId, body.runId); + const secretHash = await verifyRunBinding(body.algorithmId, body.runId, body.runSecret); + if (!secretHash) { + return { statusCode: 403, body: "run binding invalid or expired" }; + } + + const challenge = await issueChallenge(body.algorithmId, body.runId, secretHash); return { statusCode: 200, body: JSON.stringify(challenge) }; } finally { await recordRequest(body.algorithmId, body.runId, CostPhase.Execution, startedAt); diff --git a/secure/verifier-service/src/handlers/registerRun.ts b/secure/verifier-service/src/handlers/registerRun.ts new file mode 100644 index 0000000..1b5512b --- /dev/null +++ b/secure/verifier-service/src/handlers/registerRun.ts @@ -0,0 +1,40 @@ +import { CostPhase, recordRequest } from "@srads/common/cost"; +import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; +import { registerRunBinding } from "../lib/runBinding.js"; + +interface RegisterRunBody { + algorithmId: string; + runId: string; + runSecret: string; +} + +function isRegisterRunBody(body: unknown): body is RegisterRunBody { + if (typeof body !== "object" || body === null) return false; + const b = body as Record; + return [b.algorithmId, b.runId, b.runSecret].every((value) => typeof value === "string" && value.length > 0); +} + +export const handler: APIGatewayProxyHandlerV2 = async (event) => { + const startedAt = process.hrtime.bigint(); + if (!event.body) { + return { statusCode: 400, body: "missing request body" }; + } + + let body: unknown; + try { + body = JSON.parse(event.body); + } catch { + return { statusCode: 400, body: "invalid JSON" }; + } + + if (!isRegisterRunBody(body)) { + return { statusCode: 400, body: "expected algorithmId, runId, and runSecret as non-empty strings" }; + } + + try { + await registerRunBinding(body.algorithmId, body.runId, body.runSecret); + return { statusCode: 201, body: JSON.stringify({ runId: body.runId }) }; + } finally { + await recordRequest(body.algorithmId, body.runId, CostPhase.Execution, startedAt); + } +}; diff --git a/secure/verifier-service/src/lib/attestFlow.ts b/secure/verifier-service/src/lib/attestFlow.ts index f5e08e1..477e8c6 100644 --- a/secure/verifier-service/src/lib/attestFlow.ts +++ b/secure/verifier-service/src/lib/attestFlow.ts @@ -1,5 +1,6 @@ import { timingSafeEqual } from "node:crypto"; import { type AttestationDocument, pcrHex } from "./attestation.js"; +import type { ConsumedChallenge } from "./challenge.js"; export interface AttestBody { algorithmId: string; @@ -17,7 +18,7 @@ export interface RegisteredPcrs { export interface AttestDeps { registered: RegisteredPcrs; verifyDocument(raw: Buffer): AttestationDocument; - consumeChallenge(challenge: string, algorithmId: string, runId: string): Promise; + consumeChallenge(challenge: string, algorithmId: string, runId: string): Promise; issueTokens(algorithmId: string, runId: string): Promise; } @@ -33,8 +34,16 @@ function decodeChallenge(challenge: string): Buffer | null { return Buffer.from(challenge, "base64url"); } +function userDataMatches(userData: Buffer | undefined, secretHashHex: string): boolean { + if (!userData) return false; + const expected = Buffer.from(secretHashHex, "hex"); + return userData.length === expected.length && timingSafeEqual(userData, expected); +} + // The freshness check runs before the PCR check, and consumes the challenge, so the same document cannot be -// replayed even when its measurements are valid. +// replayed even when its measurements are valid. The user_data check is what ties the attested session to the +// run the host was actually launched for: the enclave sets it to SHA-256(runSecret), and only that run's host +// was given the secret. export async function attestRequest(body: AttestBody, deps: AttestDeps): Promise { let doc: AttestationDocument; try { @@ -51,10 +60,15 @@ export async function attestRequest(body: AttestBody, deps: AttestDeps): Promise }; } - if (!(await deps.consumeChallenge(body.challenge, body.algorithmId, body.runId))) { + const consumed = await deps.consumeChallenge(body.challenge, body.algorithmId, body.runId); + if (!consumed) { return { statusCode: 401, body: "challenge invalid, expired, or already used" }; } + if (!userDataMatches(doc.userData, consumed.secretHash)) { + return { statusCode: 401, body: "attestation user data does not match the run binding" }; + } + const gotPcr0 = pcrHex(doc.pcrs, 0); const gotPcr1 = pcrHex(doc.pcrs, 1); const gotPcr2 = pcrHex(doc.pcrs, 2); diff --git a/secure/verifier-service/src/lib/challenge.ts b/secure/verifier-service/src/lib/challenge.ts index 04abe9a..1986c5a 100644 --- a/secure/verifier-service/src/lib/challenge.ts +++ b/secure/verifier-service/src/lib/challenge.ts @@ -21,7 +21,7 @@ function challengeId(challenge: string): string { return createHash("sha256").update(challenge).digest("hex"); } -export async function issueChallenge(algorithmId: string, runId: string): Promise { +export async function issueChallenge(algorithmId: string, runId: string, secretHash: string): Promise { const challenge = randomBytes(32).toString("base64url"); const now = Date.now(); const expiresAt = now + CHALLENGE_TTL_SECONDS * 1000; @@ -33,6 +33,7 @@ export async function issueChallenge(algorithmId: string, runId: string): Promis ChallengeId: challengeId(challenge), AlgorithmId: algorithmId, RunId: runId, + SecretHash: secretHash, ExpiresAt: Math.floor(expiresAt / 1000), CreatedAt: new Date(now).toISOString(), }, @@ -51,9 +52,18 @@ export async function issueChallenge(algorithmId: string, runId: string): Promis return { challenge, expiresAt: new Date(expiresAt).toISOString() }; } +export interface ConsumedChallenge { + secretHash: string; +} + // Atomically checks the challenge belongs to this algorithm/run, is unexpired and unused, then marks it used. -// A second call with the same challenge fails the condition, which is what rejects a replay. -export async function consumeChallenge(challenge: string, algorithmId: string, runId: string): Promise { +// A second call with the same challenge fails the condition, which is what rejects a replay. The old row is +// returned so /attest can compare the attestation document's user_data against the run's secret hash. +export async function consumeChallenge( + challenge: string, + algorithmId: string, + runId: string, +): Promise { const now = Math.floor(Date.now() / 1000); try { @@ -70,6 +80,7 @@ export async function consumeChallenge(challenge: string, algorithmId: string, r ":runId": runId, ":now": now, }, + ReturnValues: "ALL_OLD", ReturnConsumedCapacity: "TOTAL", }), ); @@ -81,9 +92,10 @@ export async function consumeChallenge(challenge: string, algorithmId: string, r "write-capacity", write.ConsumedCapacity?.CapacityUnits ?? 0, ); - return true; + const secretHash = write.Attributes?.SecretHash; + return typeof secretHash === "string" ? { secretHash } : null; } catch (err) { - if ((err as Error).name === "ConditionalCheckFailedException") return false; + if ((err as Error).name === "ConditionalCheckFailedException") return null; throw err; } } diff --git a/secure/verifier-service/src/lib/runBinding.ts b/secure/verifier-service/src/lib/runBinding.ts new file mode 100644 index 0000000..0484da9 --- /dev/null +++ b/secure/verifier-service/src/lib/runBinding.ts @@ -0,0 +1,69 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient, GetCommand, PutCommand } from "@aws-sdk/lib-dynamodb"; +import { CostPhase, costEvent } from "@srads/common/cost"; +import { requiredEnv } from "@srads/common/env"; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const tableName = requiredEnv("RUN_BINDING_TABLE"); + +// An attestation happens within minutes of launch; an hour covers boot and EIF pull without keeping a run's +// secret hash around longer than the run itself. +const RUN_BINDING_TTL_SECONDS = 60 * 60; + +function hashSecret(secret: string): string { + return createHash("sha256").update(secret).digest("hex"); +} + +export async function registerRunBinding(algorithmId: string, runId: string, runSecret: string): Promise { + const write = await ddb.send( + new PutCommand({ + TableName: tableName, + Item: { + RunId: runId, + AlgorithmId: algorithmId, + SecretHash: hashSecret(runSecret), + ExpiresAt: Math.floor(Date.now() / 1000) + RUN_BINDING_TTL_SECONDS, + CreatedAt: new Date().toISOString(), + }, + ReturnConsumedCapacity: "TOTAL", + }), + ); + await costEvent( + algorithmId, + runId, + CostPhase.Execution, + "dynamodb", + "write-capacity", + write.ConsumedCapacity?.CapacityUnits ?? 0, + ); +} + +// Returns the stored secret hash on a match, so /challenge can carry it forward to /attest without a second +// table read. Null covers a missing row, a different algorithm, an expired row, or a wrong secret alike. +export async function verifyRunBinding(algorithmId: string, runId: string, runSecret: string): Promise { + const read = await ddb.send( + new GetCommand({ + TableName: tableName, + Key: { RunId: runId }, + ReturnConsumedCapacity: "TOTAL", + }), + ); + await costEvent( + algorithmId, + runId, + CostPhase.Execution, + "dynamodb", + "read-capacity", + read.ConsumedCapacity?.CapacityUnits ?? 0, + ); + + const item = read.Item; + if (!item || item.AlgorithmId !== algorithmId) return null; + if (typeof item.ExpiresAt !== "number" || item.ExpiresAt <= Math.floor(Date.now() / 1000)) return null; + if (typeof item.SecretHash !== "string") return null; + + const expected = Buffer.from(item.SecretHash, "hex"); + const got = Buffer.from(hashSecret(runSecret), "hex"); + return expected.length === got.length && timingSafeEqual(expected, got) ? item.SecretHash : null; +}