diff --git a/rads/log-service/README.md b/rads/log-service/README.md index 0e7ff12..abd0020 100644 --- a/rads/log-service/README.md +++ b/rads/log-service/README.md @@ -6,13 +6,13 @@ Run and debug events live in [`../telemetry-service`](../telemetry-service). ## Endpoints -- `POST /subject-access` - the executor posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after the DC app produced a result. +- `POST /subject-access` - the executor posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after the DC app produced a result. If any subject in the batch already has a row for that run, the handler returns `409`. - `GET /subject-access/{dataSubjectId}` - one data subject's chronological access entries as a key query. - `GET /subject-access` - a cursor-paginated view over all access events for the authority dashboard. `?limit=` caps at 1000 (default 500); pass the returned `nextCursor` as `?cursor=` until it is `null`. Scan order is not chronological. ## Key model -Partition key `DataSubjectId`, sort key `Sk = ##`, so the write is idempotent and a data-subject lookup is a key query rather than a scan. +Partition key `DataSubjectId`, sort key `Sk = ##`, so a data-subject lookup is a key query rather than a scan. A repeat write for the same `(algorithmId, runId)` is rejected with `409` rather than appended under a new timestamp. ## Deploy diff --git a/rads/log-service/src/handlers/subjectAccessIngest.ts b/rads/log-service/src/handlers/subjectAccessIngest.ts index a59b71e..00839c7 100644 --- a/rads/log-service/src/handlers/subjectAccessIngest.ts +++ b/rads/log-service/src/handlers/subjectAccessIngest.ts @@ -4,6 +4,7 @@ import { requiredEnv } from "@srads/common/env"; import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { retryAsync } from "ts-retry"; import { parseSubjectAccessBody, toItem } from "../lib/subjectAccess.js"; +import { hasExistingAccess } from "../lib/subjectAccessDuplicates.js"; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); const table = requiredEnv("SUBJECT_ACCESS_TABLE"); @@ -48,9 +49,14 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { const parsed = parseSubjectAccessBody(body); if ("error" in parsed) return jsonResponse(400, { error: parsed.error }); + const dataSubjectIds = [...new Set(parsed.dataSubjectIds)]; + if (await hasExistingAccess(ddb, table, parsed.algorithmId, parsed.runId, dataSubjectIds)) { + return jsonResponse(409, { error: "subject access already recorded for this run" }); + } + // A DS-keyed subject-access row per data subject; the sort key is (timestamp, algorithmId, runId). // BatchWrite takes WriteRequests, not bare items: the document client only marshals PutRequest.Item. - const items = [...new Set(parsed.dataSubjectIds)].map((dataSubjectId) => ({ + const items = dataSubjectIds.map((dataSubjectId) => ({ PutRequest: { Item: toItem({ dataSubjectId, diff --git a/rads/log-service/src/lib/subjectAccessDuplicates.ts b/rads/log-service/src/lib/subjectAccessDuplicates.ts new file mode 100644 index 0000000..d396a01 --- /dev/null +++ b/rads/log-service/src/lib/subjectAccessDuplicates.ts @@ -0,0 +1,41 @@ +import { type DynamoDBDocumentClient, paginateQuery } from "@aws-sdk/lib-dynamodb"; + +const CHUNK = 25; + +async function subjectHasAccess( + ddb: DynamoDBDocumentClient, + table: string, + algorithmId: string, + runId: string, + dataSubjectId: string, +): Promise { + const pages = paginateQuery( + { client: ddb }, + { + TableName: table, + KeyConditionExpression: "DataSubjectId = :s", + FilterExpression: "AlgorithmId = :a AND RunId = :r", + ExpressionAttributeValues: { ":s": dataSubjectId, ":a": algorithmId, ":r": runId }, + }, + ); + for await (const page of pages) { + if ((page.Items ?? []).length > 0) return true; + } + return false; +} + +export async function hasExistingAccess( + ddb: DynamoDBDocumentClient, + table: string, + algorithmId: string, + runId: string, + dataSubjectIds: string[], +): Promise { + for (let i = 0; i < dataSubjectIds.length; i += CHUNK) { + const found = await Promise.all( + dataSubjectIds.slice(i, i + CHUNK).map((id) => subjectHasAccess(ddb, table, algorithmId, runId, id)), + ); + if (found.includes(true)) return true; + } + return false; +} diff --git a/rads/telemetry-service/README.md b/rads/telemetry-service/README.md index e17750e..32446a8 100644 --- a/rads/telemetry-service/README.md +++ b/rads/telemetry-service/README.md @@ -6,7 +6,7 @@ The data-subject access trail lives in [`../log-service`](../log-service). ## Endpoints -- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. +- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. If that run already recorded the same event, the handler returns `409` instead of enqueuing another row. - `GET /logs/{algorithmId}` - exports one algorithm's timeline. `?runId=` narrows to a single run. - `GET /logs` - exports the whole table. diff --git a/rads/telemetry-service/src/handlers/events.ts b/rads/telemetry-service/src/handlers/events.ts index e2dec0f..0924a40 100644 --- a/rads/telemetry-service/src/handlers/events.ts +++ b/rads/telemetry-service/src/handlers/events.ts @@ -2,6 +2,7 @@ import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; import { requiredEnv } from "@srads/common/env"; import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { parseIngestBody } from "../lib/ingest.js"; +import { runEventExists } from "../lib/lookup.js"; const sqs = new SQSClient({}); const queueUrl = requiredEnv("EVENT_QUEUE_URL"); @@ -21,6 +22,10 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { const parsed = parseIngestBody(body); if ("error" in parsed) return jsonResponse(400, { error: parsed.error }); + if (parsed.runId && (await runEventExists(parsed.algorithmId, parsed.runId, parsed.event))) { + return jsonResponse(409, { error: "event already recorded for this run" }); + } + await sqs.send(new SendMessageCommand({ QueueUrl: queueUrl, MessageBody: JSON.stringify(parsed) })); return jsonResponse(202, { ok: true }); }; diff --git a/rads/telemetry-service/src/lib/lookup.ts b/rads/telemetry-service/src/lib/lookup.ts new file mode 100644 index 0000000..d06029c --- /dev/null +++ b/rads/telemetry-service/src/lib/lookup.ts @@ -0,0 +1,23 @@ +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb"; +import { requiredEnv } from "@srads/common/env"; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const table = requiredEnv("RUN_LOG_TABLE"); + +export async function runEventExists(algorithmId: string, runId: string, event: string): Promise { + let exclusiveStartKey: Record | undefined; + do { + const result = await ddb.send( + new QueryCommand({ + TableName: table, + KeyConditionExpression: "AlgorithmId = :a AND begins_with(Sk, :s)", + ExpressionAttributeValues: { ":a": algorithmId, ":s": `${runId}#` }, + ExclusiveStartKey: exclusiveStartKey, + }), + ); + if ((result.Items ?? []).some((item) => item.Event === event)) return true; + exclusiveStartKey = result.LastEvaluatedKey; + } while (exclusiveStartKey); + return false; +} diff --git a/secure/log-service/README.md b/secure/log-service/README.md index f9c8c2d..52ba1d7 100644 --- a/secure/log-service/README.md +++ b/secure/log-service/README.md @@ -8,7 +8,7 @@ Run, build, and debug events live in [`../telemetry-service`](../telemetry-servi The public domain carries the subject-access write and read paths: -- `POST /subject-access` - sce-unit posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after a successful DC app run, using the ids PDM returned for that run. The bearer token is verified, so only an attested enclave can write its own algorithm's trail. +- `POST /subject-access` - sce-unit posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after a successful DC app run, using the ids PDM returned for that run. The bearer token is verified, so only an attested enclave can write its own algorithm's trail. If any subject in the batch already has a row for that run, the handler returns `409`. - `GET /subject-access/{dataSubjectId}` - one data subject's chronological access entries as a key query. - `GET /subject-access` - a cursor-paginated view over all access events for the authority dashboard. `?limit=` caps at 1000 (default 500); pass the returned `nextCursor` as `?cursor=` until it is `null`. Scan order is not chronological. @@ -20,7 +20,7 @@ The `rads` baseline has no verifier, so its log-service accepts subject-access w ## Key model -Partition key `DataSubjectId`, sort key `Sk = ##`. That tuple is unique per item: a data subject's record lives on exactly one PDM page, so it is read once per run by exactly one algorithm. A redelivered ingest overwrites an identical row instead of appending a duplicate. +Partition key `DataSubjectId`, sort key `Sk = ##`. That tuple is unique per item: a data subject's record lives on exactly one PDM page, so it is read once per run by exactly one algorithm. A repeat write for the same `(algorithmId, runId)` is rejected with `409` rather than appended under a new timestamp. ## Prerequisites (manual, one-time, outside this repo's IaC) diff --git a/secure/log-service/src/handlers/subjectAccessIngest.ts b/secure/log-service/src/handlers/subjectAccessIngest.ts index 681646e..b6e118c 100644 --- a/secure/log-service/src/handlers/subjectAccessIngest.ts +++ b/secure/log-service/src/handlers/subjectAccessIngest.ts @@ -5,6 +5,7 @@ import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { retryAsync } from "ts-retry"; import { requireEnclaveToken } from "../lib/auth.js"; import { parseSubjectAccessBody, toItem } from "../lib/subjectAccess.js"; +import { hasExistingAccess } from "../lib/subjectAccessDuplicates.js"; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); const table = requiredEnv("SUBJECT_ACCESS_TABLE"); @@ -54,9 +55,14 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { const authError = await requireEnclaveToken(event.headers.authorization, parsed.algorithmId, parsed.runId); if (authError) return jsonResponse(401, { error: authError }); + const dataSubjectIds = [...new Set(parsed.dataSubjectIds)]; + if (await hasExistingAccess(ddb, table, parsed.algorithmId, parsed.runId, dataSubjectIds)) { + return jsonResponse(409, { error: "subject access already recorded for this run" }); + } + // A DS-keyed subject-access row per data subject; the sort key is (timestamp, algorithmId, runId). // BatchWrite takes WriteRequests, not bare items: the document client only marshals PutRequest.Item. - const items = [...new Set(parsed.dataSubjectIds)].map((dataSubjectId) => ({ + const items = dataSubjectIds.map((dataSubjectId) => ({ PutRequest: { Item: toItem({ dataSubjectId, diff --git a/secure/log-service/src/lib/subjectAccessDuplicates.ts b/secure/log-service/src/lib/subjectAccessDuplicates.ts new file mode 100644 index 0000000..d396a01 --- /dev/null +++ b/secure/log-service/src/lib/subjectAccessDuplicates.ts @@ -0,0 +1,41 @@ +import { type DynamoDBDocumentClient, paginateQuery } from "@aws-sdk/lib-dynamodb"; + +const CHUNK = 25; + +async function subjectHasAccess( + ddb: DynamoDBDocumentClient, + table: string, + algorithmId: string, + runId: string, + dataSubjectId: string, +): Promise { + const pages = paginateQuery( + { client: ddb }, + { + TableName: table, + KeyConditionExpression: "DataSubjectId = :s", + FilterExpression: "AlgorithmId = :a AND RunId = :r", + ExpressionAttributeValues: { ":s": dataSubjectId, ":a": algorithmId, ":r": runId }, + }, + ); + for await (const page of pages) { + if ((page.Items ?? []).length > 0) return true; + } + return false; +} + +export async function hasExistingAccess( + ddb: DynamoDBDocumentClient, + table: string, + algorithmId: string, + runId: string, + dataSubjectIds: string[], +): Promise { + for (let i = 0; i < dataSubjectIds.length; i += CHUNK) { + const found = await Promise.all( + dataSubjectIds.slice(i, i + CHUNK).map((id) => subjectHasAccess(ddb, table, algorithmId, runId, id)), + ); + if (found.includes(true)) return true; + } + return false; +} diff --git a/secure/telemetry-service/README.md b/secure/telemetry-service/README.md index 393a5b1..a1d1722 100644 --- a/secure/telemetry-service/README.md +++ b/secure/telemetry-service/README.md @@ -4,7 +4,7 @@ Serverless Framework v4, TypeScript. Diagnostic timeline for performance, test, ## Endpoints -- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. No token and no client certificate. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. +- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. No token and no client certificate. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. If that run already recorded the same event, the handler returns `409` instead of enqueuing another row. - `GET /logs/{algorithmId}` - exports one algorithm's timeline. `?runId=` narrows to a single run. Pages through all results. - `GET /logs` - exports the whole table across all algorithms. diff --git a/secure/telemetry-service/src/handlers/events.ts b/secure/telemetry-service/src/handlers/events.ts index b06caf7..c048495 100644 --- a/secure/telemetry-service/src/handlers/events.ts +++ b/secure/telemetry-service/src/handlers/events.ts @@ -1,5 +1,6 @@ import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { parseIngestBody } from "../lib/ingest.js"; +import { runEventExists } from "../lib/lookup.js"; import { enqueueEvent } from "../lib/queue.js"; function jsonResponse(statusCode: number, body: unknown) { @@ -19,6 +20,10 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { const parsed = parseIngestBody(body); if ("error" in parsed) return jsonResponse(400, { error: parsed.error }); + if (parsed.runId && (await runEventExists(parsed.algorithmId, parsed.runId, parsed.event))) { + return jsonResponse(409, { error: "event already recorded for this run" }); + } + await enqueueEvent(parsed); return jsonResponse(202, { ok: true }); }; diff --git a/secure/telemetry-service/src/lib/lookup.ts b/secure/telemetry-service/src/lib/lookup.ts new file mode 100644 index 0000000..d06029c --- /dev/null +++ b/secure/telemetry-service/src/lib/lookup.ts @@ -0,0 +1,23 @@ +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb"; +import { requiredEnv } from "@srads/common/env"; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const table = requiredEnv("RUN_LOG_TABLE"); + +export async function runEventExists(algorithmId: string, runId: string, event: string): Promise { + let exclusiveStartKey: Record | undefined; + do { + const result = await ddb.send( + new QueryCommand({ + TableName: table, + KeyConditionExpression: "AlgorithmId = :a AND begins_with(Sk, :s)", + ExpressionAttributeValues: { ":a": algorithmId, ":s": `${runId}#` }, + ExclusiveStartKey: exclusiveStartKey, + }), + ); + if ((result.Items ?? []).some((item) => item.Event === event)) return true; + exclusiveStartKey = result.LastEvaluatedKey; + } while (exclusiveStartKey); + return false; +}