Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ TRANSPARENCY_DOMAIN_SECURE_RADS=transparency.sr.secure-rads.org
LOG_HOST_RADS=log.r.secure-rads.org
LOG_HOST_SECURE_RADS=log.sr.secure-rads.org
# mTLS domain for internal producers (registry, execution-service, result-consumer).
LOG_MTLS_HOST_SECURE_RADS=log-mtls.sr.secure-rads.org
TELEMETRY_HOST_RADS=telemetry.r.secure-rads.org
TELEMETRY_HOST_SECURE_RADS=telemetry.sr.secure-rads.org
COST_HOST_RADS=cost.r.secure-rads.org
COST_HOST_SECURE_RADS=cost.sr.secure-rads.org
PDM_HOST=pdm.sr.secure-rads.org
Expand All @@ -39,7 +40,7 @@ VERIFIER_HOST_PUBKEY_SHA256=
PDM_HOST_PUBKEY_SHA256=
KMS_HOST_PUBKEY_SHA256=
LOG_HOST_PUBKEY_SHA256=
LOG_MTLS_PUBKEY_SHA256=
TELEMETRY_HOST_PUBKEY_SHA256=
VERIFIER_JWKS_URL=https://verifier-attest.sr.secure-rads.org/.well-known/jwks.json
# SHA-256 of the verifier JWKS public key, printed by the verifier-service deploy.
# Used by pdm-service and log-service to verify attested access tokens.
Expand Down
16 changes: 16 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ jobs:
- 'rads/log-service/**'
- 'packages/common/**'
- '.github/workflows/ci.yml'
'rads/telemetry-service':
- 'rads/telemetry-service/**'
- 'packages/common/**'
- '.github/workflows/ci.yml'
'rads/pdm-service':
- 'rads/pdm-service/**'
- 'packages/common/**'
Expand All @@ -64,6 +68,10 @@ jobs:
- 'secure/log-service/**'
- 'packages/common/**'
- '.github/workflows/ci.yml'
'secure/telemetry-service':
- 'secure/telemetry-service/**'
- 'packages/common/**'
- '.github/workflows/ci.yml'
'secure/pdm-service':
- 'secure/pdm-service/**'
- 'packages/common/**'
Expand Down Expand Up @@ -105,6 +113,10 @@ jobs:
- 'rads/log-service/**'
- 'packages/common/**'
- '.github/workflows/ci.yml'
'rads/telemetry-service':
- 'rads/telemetry-service/**'
- 'packages/common/**'
- '.github/workflows/ci.yml'
'rads/pdm-service':
- 'rads/pdm-service/**'
- 'packages/common/**'
Expand All @@ -125,6 +137,10 @@ jobs:
- 'secure/log-service/**'
- 'packages/common/**'
- '.github/workflows/ci.yml'
'secure/telemetry-service':
- 'secure/telemetry-service/**'
- 'packages/common/**'
- '.github/workflows/ci.yml'
'secure/pdm-service':
- 'secure/pdm-service/**'
- 'packages/common/**'
Expand Down
11 changes: 6 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ logs:
pids=""; \
for target in registry-logs; do make $$target & pids="$$pids $$!"; done; \
for group in $$(aws logs describe-log-groups --region "$(REGION)" --log-group-name-prefix /aws/lambda/ --query 'logGroups[].logGroupName' --output text); do \
case "$$group" in /aws/lambda/secure-rads-execution-service-dev-*|/aws/lambda/secure-rads-verifier-service-dev-*|/aws/lambda/secure-rads-pdm-service-dev-*|/aws/lambda/secure-rads-kms-service-dev-*|/aws/lambda/secure-rads-log-service-dev-*|/aws/lambda/secure-rads-cost-service-dev-*|/aws/lambda/secure-rads-result-consumer-dev-*|/aws/lambda/rads-registry-service-dev-*|/aws/lambda/rads-pdm-service-dev-*|/aws/lambda/rads-log-service-dev-*|/aws/lambda/rads-cost-service-dev-*|/aws/lambda/rads-result-consumer-dev-*|/aws/lambda/rads-execution-service-dev-*) aws logs tail "$$group" --region "$(REGION)" --follow & pids="$$pids $$!";; esac; \
case "$$group" in /aws/lambda/secure-rads-execution-service-dev-*|/aws/lambda/secure-rads-verifier-service-dev-*|/aws/lambda/secure-rads-pdm-service-dev-*|/aws/lambda/secure-rads-kms-service-dev-*|/aws/lambda/secure-rads-log-service-dev-*|/aws/lambda/secure-rads-telemetry-service-dev-*|/aws/lambda/secure-rads-cost-service-dev-*|/aws/lambda/secure-rads-result-consumer-dev-*|/aws/lambda/rads-registry-service-dev-*|/aws/lambda/rads-pdm-service-dev-*|/aws/lambda/rads-log-service-dev-*|/aws/lambda/rads-telemetry-service-dev-*|/aws/lambda/rads-cost-service-dev-*|/aws/lambda/rads-result-consumer-dev-*|/aws/lambda/rads-execution-service-dev-*) aws logs tail "$$group" --region "$(REGION)" --follow & pids="$$pids $$!";; esac; \
done; \
trap 'kill $$pids 2>/dev/null || true' EXIT INT TERM; \
for pid in $$pids; do wait "$$pid" || true; done
Expand Down Expand Up @@ -85,6 +85,7 @@ REGION := $(AWS_REGION)
$(call env_require,AWS_REGION)
KEY_PATH := $(CURDIR)/.ssh/csav-key.pem
LOG_HOST = $(LOG_HOST_SECURE_RADS)
TELEMETRY_HOST = $(TELEMETRY_HOST_SECURE_RADS)
COST_HOST = $(COST_HOST_SECURE_RADS)
instance_ip = $(shell aws ec2 describe-instances --region $(REGION) --instance-ids $(shell aws cloudformation describe-stacks \
--stack-name $(1) --region $(REGION) --query "Stacks[0].Outputs[?OutputKey=='$(2)'].OutputValue" --output text) \
Expand Down Expand Up @@ -326,26 +327,26 @@ rads-all:

# Show the rads serverless stack outputs (API URLs and resource names).
rads-status:
@for stack in rads-registry-service-dev rads-pdm-service-dev rads-log-service-dev rads-cost-service-dev rads-result-consumer-dev rads-execution-service-dev; do \
@for stack in rads-registry-service-dev rads-pdm-service-dev rads-log-service-dev rads-telemetry-service-dev rads-cost-service-dev rads-result-consumer-dev rads-execution-service-dev; do \
aws cloudformation describe-stacks --stack-name $$stack --region $(REGION) --query 'Stacks[0].Outputs' --output table; \
done

# Export one run's timeline to run-<RUN_ID>.csv. Needs both ids (the log table is keyed by algorithm).
logs-csv:
@test -n "$(ALGORITHM_ID)" || { echo "ALGORITHM_ID is required"; exit 1; }
@test -n "$(RUN_ID)" || { echo "RUN_ID is required"; exit 1; }
curl -sS "https://$(LOG_HOST)/logs/$(ALGORITHM_ID)?runId=$(RUN_ID)&format=csv" -o run-$(RUN_ID).csv
curl -sS "https://$(TELEMETRY_HOST)/logs/$(ALGORITHM_ID)?runId=$(RUN_ID)&format=csv" -o run-$(RUN_ID).csv
@echo "wrote run-$(RUN_ID).csv"

# Export one algorithm's whole timeline to algo-<ALGORITHM_ID>.csv.
logs-csv-algo:
@test -n "$(ALGORITHM_ID)" || { echo "ALGORITHM_ID is required"; exit 1; }
curl -sS "https://$(LOG_HOST)/logs/$(ALGORITHM_ID)?format=csv" -o algo-$(ALGORITHM_ID).csv
curl -sS "https://$(TELEMETRY_HOST)/logs/$(ALGORITHM_ID)?format=csv" -o algo-$(ALGORITHM_ID).csv
@echo "wrote algo-$(ALGORITHM_ID).csv"

# Export the entire log table across all algorithms to logs-all.csv.
logs-csv-all:
curl -sS "https://$(LOG_HOST)/logs?format=csv" -o logs-all.csv
curl -sS "https://$(TELEMETRY_HOST)/logs?format=csv" -o logs-all.csv
@echo "wrote logs-all.csv"

# Follow x-next-cursor to pull every page of a paginated cost CSV export into OUT.
Expand Down
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ attestation, and blue is data-key release.
- **Algorithm Registry (AR)** holds algorithms that were vetted and validated against a consent, and registers their measurements with the Verifier. In this PoC, registry-service performs the registration; algorithm vetting and consent approval are out of scope.
- **Verifier** stores the expected measurements per algorithm and issues a signed, time-constrained token to an SCE that attests correctly.
- **Key Management Service (KMS)** releases the data-encryption key only to an SCE holding a valid verifier token.
- **Log Service** holds the processing records that support DS and Supervisory Authority queries.
- **Log Service** holds the data-subject access trail that supports DS and Supervisory Authority queries.
- **Telemetry Service** holds the best-effort build and run timeline used for performance, debugging, and benchmark exports.
- **Secure Computing Environment (SCE)** fetches encrypted records, decrypts them, runs the DC's algorithm inside the enclave, and records the run with the Log Service.

This repo implements the SCE and the supporting services needed to run and inspect it. Consent capture,
Expand All @@ -37,7 +38,7 @@ service.

## rads (baseline)

[`rads/`](rads/) is the fully-trusted baseline sibling: no KMS, no Nitro enclave, no verifier. A data controller's docker image tarball is uploaded to the registry-service (`rads/csav/registry-service`), stored raw in S3, and recorded as `algorithmId -> S3 path` in DynamoDB. At run time the execution-service (`rads/sce/execution-service`) launches a plain EC2 executor that docker-loads the image, fetches records from PDM into `records.json`, and runs the app with `RECORDS_PATH` / `RESULT_PATH`. The result file is delivered asynchronously to result-consumer; stdout is the diagnostic fallback. The same log-service records the run timeline.
[`rads/`](rads/) is the fully-trusted baseline sibling: no KMS, no Nitro enclave, no verifier. A data controller's docker image tarball is uploaded to the registry-service (`rads/csav/registry-service`), stored raw in S3, and recorded as `algorithmId -> S3 path` in DynamoDB. At run time the execution-service (`rads/sce/execution-service`) launches a plain EC2 executor that docker-loads the image, fetches records from PDM into `records.json`, and runs the app with `RECORDS_PATH` / `RESULT_PATH`. The result file is delivered asynchronously to result-consumer; stdout is the diagnostic fallback. The same telemetry-service records the run timeline.

Every rads AWS resource is prefixed `rads-`; every secure-rads resource is prefixed `secure-rads-`, so both systems deploy and run side by side in one account/region.

Expand All @@ -47,7 +48,7 @@ Every rads AWS resource is prefixed `rads-`; every secure-rads resource is prefi

**Execution phase (SCE).** A caller POSTs an `algorithmId` to the API Gateway and Lambda execution-service control plane, which launches a per-run Nitro Enclave parent EC2 instance. Inside, the SCE Unit seeds enclave entropy from the NSM, attests to the Verifier Service, and - once attestation passes - fetches that algorithm's records from the PDM Service, gets the data key unwrapped by the KMS Service, decrypts them, and writes `records.json` for the jailed Data Controller app (`RECORDS_PATH` / `RESULT_PATH`, same file contract as rads). Results come back asynchronously: `/run` returns a `runId` immediately, the enclave host publishes the result to SQS, a consumer Lambda persists it, and the caller polls `GET /run/{runId}`.

Throughout both phases every component posts timestamped events to the log-service, so a run's full timeline - build to execution - can be exported for benchmarking. Components also report selected, attributable AWS operations to the cost-service, keyed by the same `(AlgorithmId, runId)`, so their estimated costs can be summed per AWS service ([`docs/cost-service.md`](docs/cost-service.md)). Event-by-event plaintext flows: [`docs/e2e-rads.txt`](docs/e2e-rads.txt), [`docs/e2e-secure-rads.txt`](docs/e2e-secure-rads.txt).
Throughout both phases every component posts timestamped events to the telemetry-service, so a run's full timeline - build to execution - can be exported for benchmarking. The log-service records only the subject-access trail. Components also report selected, attributable AWS operations to the cost-service, keyed by the same `(AlgorithmId, runId)`, so their estimated costs can be summed per AWS service ([`docs/cost-service.md`](docs/cost-service.md)). Event-by-event plaintext flows: [`docs/e2e-rads.txt`](docs/e2e-rads.txt), [`docs/e2e-secure-rads.txt`](docs/e2e-secure-rads.txt).

## Quick start

Expand Down
4 changes: 2 additions & 2 deletions benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ Each sweep writes `results/raw/sweep-<unix>.json`. Stop with Ctrl-C and terminat
## Export and figures

```sh
make export # every results/raw/sweep-*.json + log-service + cost-service -> phase-timings.csv
make export # every results/raw/sweep-*.json + telemetry-service + cost-service -> phase-timings.csv
make plot # -> results/figures/sweep-phases.png, sweep-scaling.png, sweep-cost.png, sweep-cost-services.png
make report # both
```
Expand Down Expand Up @@ -138,7 +138,7 @@ Existing committed results use the earlier event contract (`data-decrypted` afte
| --- | --- |
| `Makefile` | `seed` / `run` / `export` / `plot` / `report` / `all` |
| `src/execute.py` | Setup and size sweep (`--setup-only`, `--skip-*`) |
| `src/export.py` | Raw JSON + log-service → `phase-timings.csv` |
| `src/export.py` | Raw JSON + telemetry-service → `phase-timings.csv` |
| `src/plot.py` | CSV → phase / scaling figures |
| `src/lib/catalog.py` | `ApiGatewayService` / `Ec2HttpService` catalog |
| `src/lib/config.py` | Trial constants + `resolve()` |
Expand Down
Binary file modified benchmark/results/figures/sweep-cost-services.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmark/results/figures/sweep-cost.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmark/results/figures/sweep-phases.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmark/results/figures/sweep-scaling.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified benchmark/results/figures/sweep-selected-phases.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Loading