Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions rads/log-service/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ Run and debug events live in [`../telemetry-service`](../telemetry-service).

## Endpoints

- `POST /subject-access` - the executor posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after the DC app produced a result.
- `POST /subject-access` - the executor posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after the DC app produced a result. If any subject in the batch already has a row for that run, the handler returns `409`.
- `GET /subject-access/{dataSubjectId}` - one data subject's chronological access entries as a key query.
- `GET /subject-access` - a cursor-paginated view over all access events for the authority dashboard. `?limit=` caps at 1000 (default 500); pass the returned `nextCursor` as `?cursor=` until it is `null`. Scan order is not chronological.

## Key model

Partition key `DataSubjectId`, sort key `Sk = <timestamp>#<algorithmId>#<runId>`, so the write is idempotent and a data-subject lookup is a key query rather than a scan.
Partition key `DataSubjectId`, sort key `Sk = <timestamp>#<algorithmId>#<runId>`, so a data-subject lookup is a key query rather than a scan. A repeat write for the same `(algorithmId, runId)` is rejected with `409` rather than appended under a new timestamp.

## Deploy

Expand Down
8 changes: 7 additions & 1 deletion rads/log-service/src/handlers/subjectAccessIngest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { requiredEnv } from "@srads/common/env";
import type { APIGatewayProxyHandlerV2 } from "aws-lambda";
import { retryAsync } from "ts-retry";
import { parseSubjectAccessBody, toItem } from "../lib/subjectAccess.js";
import { hasExistingAccess } from "../lib/subjectAccessDuplicates.js";

const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({}));
const table = requiredEnv("SUBJECT_ACCESS_TABLE");
Expand Down Expand Up @@ -48,9 +49,14 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => {
const parsed = parseSubjectAccessBody(body);
if ("error" in parsed) return jsonResponse(400, { error: parsed.error });

const dataSubjectIds = [...new Set(parsed.dataSubjectIds)];
if (await hasExistingAccess(ddb, table, parsed.algorithmId, parsed.runId, dataSubjectIds)) {
return jsonResponse(409, { error: "subject access already recorded for this run" });
}

// A DS-keyed subject-access row per data subject; the sort key is (timestamp, algorithmId, runId).
// BatchWrite takes WriteRequests, not bare items: the document client only marshals PutRequest.Item.
const items = [...new Set(parsed.dataSubjectIds)].map((dataSubjectId) => ({
const items = dataSubjectIds.map((dataSubjectId) => ({
PutRequest: {
Item: toItem({
dataSubjectId,
Expand Down
41 changes: 41 additions & 0 deletions rads/log-service/src/lib/subjectAccessDuplicates.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import { type DynamoDBDocumentClient, paginateQuery } from "@aws-sdk/lib-dynamodb";

const CHUNK = 25;

async function subjectHasAccess(
ddb: DynamoDBDocumentClient,
table: string,
algorithmId: string,
runId: string,
dataSubjectId: string,
): Promise<boolean> {
const pages = paginateQuery(
{ client: ddb },
{
TableName: table,
KeyConditionExpression: "DataSubjectId = :s",
FilterExpression: "AlgorithmId = :a AND RunId = :r",
ExpressionAttributeValues: { ":s": dataSubjectId, ":a": algorithmId, ":r": runId },
},
);
for await (const page of pages) {
if ((page.Items ?? []).length > 0) return true;
}
return false;
}

export async function hasExistingAccess(
ddb: DynamoDBDocumentClient,
table: string,
algorithmId: string,
runId: string,
dataSubjectIds: string[],
): Promise<boolean> {
for (let i = 0; i < dataSubjectIds.length; i += CHUNK) {
const found = await Promise.all(
dataSubjectIds.slice(i, i + CHUNK).map((id) => subjectHasAccess(ddb, table, algorithmId, runId, id)),
);
if (found.includes(true)) return true;
}
return false;
}
2 changes: 1 addition & 1 deletion rads/telemetry-service/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ The data-subject access trail lives in [`../log-service`](../log-service).

## Endpoints

- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`.
- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. If that run already recorded the same event, the handler returns `409` instead of enqueuing another row.
- `GET /logs/{algorithmId}` - exports one algorithm's timeline. `?runId=` narrows to a single run.
- `GET /logs` - exports the whole table.

Expand Down
5 changes: 5 additions & 0 deletions rads/telemetry-service/src/handlers/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
import { requiredEnv } from "@srads/common/env";
import type { APIGatewayProxyHandlerV2 } from "aws-lambda";
import { parseIngestBody } from "../lib/ingest.js";
import { runEventExists } from "../lib/lookup.js";

const sqs = new SQSClient({});
const queueUrl = requiredEnv("EVENT_QUEUE_URL");
Expand All @@ -21,6 +22,10 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => {
const parsed = parseIngestBody(body);
if ("error" in parsed) return jsonResponse(400, { error: parsed.error });

if (parsed.runId && (await runEventExists(parsed.algorithmId, parsed.runId, parsed.event))) {
return jsonResponse(409, { error: "event already recorded for this run" });
}

await sqs.send(new SendMessageCommand({ QueueUrl: queueUrl, MessageBody: JSON.stringify(parsed) }));
return jsonResponse(202, { ok: true });
};
23 changes: 23 additions & 0 deletions rads/telemetry-service/src/lib/lookup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb";
import { requiredEnv } from "@srads/common/env";

const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({}));
const table = requiredEnv("RUN_LOG_TABLE");

export async function runEventExists(algorithmId: string, runId: string, event: string): Promise<boolean> {
let exclusiveStartKey: Record<string, unknown> | undefined;
do {
const result = await ddb.send(
new QueryCommand({
TableName: table,
KeyConditionExpression: "AlgorithmId = :a AND begins_with(Sk, :s)",
ExpressionAttributeValues: { ":a": algorithmId, ":s": `${runId}#` },
ExclusiveStartKey: exclusiveStartKey,
}),
);
if ((result.Items ?? []).some((item) => item.Event === event)) return true;
exclusiveStartKey = result.LastEvaluatedKey;
} while (exclusiveStartKey);
return false;
}
4 changes: 2 additions & 2 deletions secure/log-service/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ Run, build, and debug events live in [`../telemetry-service`](../telemetry-servi

The public domain carries the subject-access write and read paths:

- `POST /subject-access` - sce-unit posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after a successful DC app run, using the ids PDM returned for that run. The bearer token is verified, so only an attested enclave can write its own algorithm's trail.
- `POST /subject-access` - sce-unit posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after a successful DC app run, using the ids PDM returned for that run. The bearer token is verified, so only an attested enclave can write its own algorithm's trail. If any subject in the batch already has a row for that run, the handler returns `409`.
- `GET /subject-access/{dataSubjectId}` - one data subject's chronological access entries as a key query.
- `GET /subject-access` - a cursor-paginated view over all access events for the authority dashboard. `?limit=` caps at 1000 (default 500); pass the returned `nextCursor` as `?cursor=` until it is `null`. Scan order is not chronological.

Expand All @@ -20,7 +20,7 @@ The `rads` baseline has no verifier, so its log-service accepts subject-access w

## Key model

Partition key `DataSubjectId`, sort key `Sk = <timestamp>#<algorithmId>#<runId>`. That tuple is unique per item: a data subject's record lives on exactly one PDM page, so it is read once per run by exactly one algorithm. A redelivered ingest overwrites an identical row instead of appending a duplicate.
Partition key `DataSubjectId`, sort key `Sk = <timestamp>#<algorithmId>#<runId>`. That tuple is unique per item: a data subject's record lives on exactly one PDM page, so it is read once per run by exactly one algorithm. A repeat write for the same `(algorithmId, runId)` is rejected with `409` rather than appended under a new timestamp.

## Prerequisites (manual, one-time, outside this repo's IaC)

Expand Down
8 changes: 7 additions & 1 deletion secure/log-service/src/handlers/subjectAccessIngest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type { APIGatewayProxyHandlerV2 } from "aws-lambda";
import { retryAsync } from "ts-retry";
import { requireEnclaveToken } from "../lib/auth.js";
import { parseSubjectAccessBody, toItem } from "../lib/subjectAccess.js";
import { hasExistingAccess } from "../lib/subjectAccessDuplicates.js";

const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({}));
const table = requiredEnv("SUBJECT_ACCESS_TABLE");
Expand Down Expand Up @@ -54,9 +55,14 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => {
const authError = await requireEnclaveToken(event.headers.authorization, parsed.algorithmId, parsed.runId);
if (authError) return jsonResponse(401, { error: authError });

const dataSubjectIds = [...new Set(parsed.dataSubjectIds)];
if (await hasExistingAccess(ddb, table, parsed.algorithmId, parsed.runId, dataSubjectIds)) {
return jsonResponse(409, { error: "subject access already recorded for this run" });
}

// A DS-keyed subject-access row per data subject; the sort key is (timestamp, algorithmId, runId).
// BatchWrite takes WriteRequests, not bare items: the document client only marshals PutRequest.Item.
const items = [...new Set(parsed.dataSubjectIds)].map((dataSubjectId) => ({
const items = dataSubjectIds.map((dataSubjectId) => ({
PutRequest: {
Item: toItem({
dataSubjectId,
Expand Down
41 changes: 41 additions & 0 deletions secure/log-service/src/lib/subjectAccessDuplicates.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import { type DynamoDBDocumentClient, paginateQuery } from "@aws-sdk/lib-dynamodb";

const CHUNK = 25;

async function subjectHasAccess(
ddb: DynamoDBDocumentClient,
table: string,
algorithmId: string,
runId: string,
dataSubjectId: string,
): Promise<boolean> {
const pages = paginateQuery(
{ client: ddb },
{
TableName: table,
KeyConditionExpression: "DataSubjectId = :s",
FilterExpression: "AlgorithmId = :a AND RunId = :r",
ExpressionAttributeValues: { ":s": dataSubjectId, ":a": algorithmId, ":r": runId },
},
);
for await (const page of pages) {
if ((page.Items ?? []).length > 0) return true;
}
return false;
}

export async function hasExistingAccess(
ddb: DynamoDBDocumentClient,
table: string,
algorithmId: string,
runId: string,
dataSubjectIds: string[],
): Promise<boolean> {
for (let i = 0; i < dataSubjectIds.length; i += CHUNK) {
const found = await Promise.all(
dataSubjectIds.slice(i, i + CHUNK).map((id) => subjectHasAccess(ddb, table, algorithmId, runId, id)),
);
if (found.includes(true)) return true;
}
return false;
}
2 changes: 1 addition & 1 deletion secure/telemetry-service/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Serverless Framework v4, TypeScript. Diagnostic timeline for performance, test,

## Endpoints

- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. No token and no client certificate. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`.
- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. No token and no client certificate. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. If that run already recorded the same event, the handler returns `409` instead of enqueuing another row.
- `GET /logs/{algorithmId}` - exports one algorithm's timeline. `?runId=` narrows to a single run. Pages through all results.
- `GET /logs` - exports the whole table across all algorithms.

Expand Down
5 changes: 5 additions & 0 deletions secure/telemetry-service/src/handlers/events.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { APIGatewayProxyHandlerV2 } from "aws-lambda";
import { parseIngestBody } from "../lib/ingest.js";
import { runEventExists } from "../lib/lookup.js";
import { enqueueEvent } from "../lib/queue.js";

function jsonResponse(statusCode: number, body: unknown) {
Expand All @@ -19,6 +20,10 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => {
const parsed = parseIngestBody(body);
if ("error" in parsed) return jsonResponse(400, { error: parsed.error });

if (parsed.runId && (await runEventExists(parsed.algorithmId, parsed.runId, parsed.event))) {
return jsonResponse(409, { error: "event already recorded for this run" });
}

await enqueueEvent(parsed);
return jsonResponse(202, { ok: true });
};
23 changes: 23 additions & 0 deletions secure/telemetry-service/src/lib/lookup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb";
import { requiredEnv } from "@srads/common/env";

const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({}));
const table = requiredEnv("RUN_LOG_TABLE");

export async function runEventExists(algorithmId: string, runId: string, event: string): Promise<boolean> {
let exclusiveStartKey: Record<string, unknown> | undefined;
do {
const result = await ddb.send(
new QueryCommand({
TableName: table,
KeyConditionExpression: "AlgorithmId = :a AND begins_with(Sk, :s)",
ExpressionAttributeValues: { ":a": algorithmId, ":s": `${runId}#` },
ExclusiveStartKey: exclusiveStartKey,
}),
);
if ((result.Items ?? []).some((item) => item.Event === event)) return true;
exclusiveStartKey = result.LastEvaluatedKey;
} while (exclusiveStartKey);
return false;
}