From d2e37470e69d75fbd7a8baa73c19d619fe76dfc5 Mon Sep 17 00:00:00 2001 From: prdai Date: Thu, 19 Nov 2026 08:25:14 +0530 Subject: [PATCH] fix(telemetry,log): reject duplicate run events Co-authored-by: Codex --- rads/log-service/README.md | 4 +- .../src/handlers/subjectAccessIngest.ts | 8 +++- .../src/lib/subjectAccessDuplicates.ts | 41 +++++++++++++++++++ rads/telemetry-service/README.md | 2 +- rads/telemetry-service/src/handlers/events.ts | 5 +++ rads/telemetry-service/src/lib/lookup.ts | 23 +++++++++++ secure/log-service/README.md | 4 +- .../src/handlers/subjectAccessIngest.ts | 8 +++- .../src/lib/subjectAccessDuplicates.ts | 41 +++++++++++++++++++ secure/telemetry-service/README.md | 2 +- .../telemetry-service/src/handlers/events.ts | 5 +++ secure/telemetry-service/src/lib/lookup.ts | 23 +++++++++++ 12 files changed, 158 insertions(+), 8 deletions(-) create mode 100644 rads/log-service/src/lib/subjectAccessDuplicates.ts create mode 100644 rads/telemetry-service/src/lib/lookup.ts create mode 100644 secure/log-service/src/lib/subjectAccessDuplicates.ts create mode 100644 secure/telemetry-service/src/lib/lookup.ts diff --git a/rads/log-service/README.md b/rads/log-service/README.md index 0e7ff124..abd0020a 100644 --- a/rads/log-service/README.md +++ b/rads/log-service/README.md @@ -6,13 +6,13 @@ Run and debug events live in [`../telemetry-service`](../telemetry-service). ## Endpoints -- `POST /subject-access` - the executor posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after the DC app produced a result. +- `POST /subject-access` - the executor posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after the DC app produced a result. If any subject in the batch already has a row for that run, the handler returns `409`. - `GET /subject-access/{dataSubjectId}` - one data subject's chronological access entries as a key query. - `GET /subject-access` - a cursor-paginated view over all access events for the authority dashboard. `?limit=` caps at 1000 (default 500); pass the returned `nextCursor` as `?cursor=` until it is `null`. Scan order is not chronological. ## Key model -Partition key `DataSubjectId`, sort key `Sk = ##`, so the write is idempotent and a data-subject lookup is a key query rather than a scan. +Partition key `DataSubjectId`, sort key `Sk = ##`, so a data-subject lookup is a key query rather than a scan. A repeat write for the same `(algorithmId, runId)` is rejected with `409` rather than appended under a new timestamp. ## Deploy diff --git a/rads/log-service/src/handlers/subjectAccessIngest.ts b/rads/log-service/src/handlers/subjectAccessIngest.ts index a59b71e8..00839c71 100644 --- a/rads/log-service/src/handlers/subjectAccessIngest.ts +++ b/rads/log-service/src/handlers/subjectAccessIngest.ts @@ -4,6 +4,7 @@ import { requiredEnv } from "@srads/common/env"; import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { retryAsync } from "ts-retry"; import { parseSubjectAccessBody, toItem } from "../lib/subjectAccess.js"; +import { hasExistingAccess } from "../lib/subjectAccessDuplicates.js"; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); const table = requiredEnv("SUBJECT_ACCESS_TABLE"); @@ -48,9 +49,14 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { const parsed = parseSubjectAccessBody(body); if ("error" in parsed) return jsonResponse(400, { error: parsed.error }); + const dataSubjectIds = [...new Set(parsed.dataSubjectIds)]; + if (await hasExistingAccess(ddb, table, parsed.algorithmId, parsed.runId, dataSubjectIds)) { + return jsonResponse(409, { error: "subject access already recorded for this run" }); + } + // A DS-keyed subject-access row per data subject; the sort key is (timestamp, algorithmId, runId). // BatchWrite takes WriteRequests, not bare items: the document client only marshals PutRequest.Item. - const items = [...new Set(parsed.dataSubjectIds)].map((dataSubjectId) => ({ + const items = dataSubjectIds.map((dataSubjectId) => ({ PutRequest: { Item: toItem({ dataSubjectId, diff --git a/rads/log-service/src/lib/subjectAccessDuplicates.ts b/rads/log-service/src/lib/subjectAccessDuplicates.ts new file mode 100644 index 00000000..d396a014 --- /dev/null +++ b/rads/log-service/src/lib/subjectAccessDuplicates.ts @@ -0,0 +1,41 @@ +import { type DynamoDBDocumentClient, paginateQuery } from "@aws-sdk/lib-dynamodb"; + +const CHUNK = 25; + +async function subjectHasAccess( + ddb: DynamoDBDocumentClient, + table: string, + algorithmId: string, + runId: string, + dataSubjectId: string, +): Promise { + const pages = paginateQuery( + { client: ddb }, + { + TableName: table, + KeyConditionExpression: "DataSubjectId = :s", + FilterExpression: "AlgorithmId = :a AND RunId = :r", + ExpressionAttributeValues: { ":s": dataSubjectId, ":a": algorithmId, ":r": runId }, + }, + ); + for await (const page of pages) { + if ((page.Items ?? []).length > 0) return true; + } + return false; +} + +export async function hasExistingAccess( + ddb: DynamoDBDocumentClient, + table: string, + algorithmId: string, + runId: string, + dataSubjectIds: string[], +): Promise { + for (let i = 0; i < dataSubjectIds.length; i += CHUNK) { + const found = await Promise.all( + dataSubjectIds.slice(i, i + CHUNK).map((id) => subjectHasAccess(ddb, table, algorithmId, runId, id)), + ); + if (found.includes(true)) return true; + } + return false; +} diff --git a/rads/telemetry-service/README.md b/rads/telemetry-service/README.md index e17750ec..32446a81 100644 --- a/rads/telemetry-service/README.md +++ b/rads/telemetry-service/README.md @@ -6,7 +6,7 @@ The data-subject access trail lives in [`../log-service`](../log-service). ## Endpoints -- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. +- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. If that run already recorded the same event, the handler returns `409` instead of enqueuing another row. - `GET /logs/{algorithmId}` - exports one algorithm's timeline. `?runId=` narrows to a single run. - `GET /logs` - exports the whole table. diff --git a/rads/telemetry-service/src/handlers/events.ts b/rads/telemetry-service/src/handlers/events.ts index e2dec0f7..0924a407 100644 --- a/rads/telemetry-service/src/handlers/events.ts +++ b/rads/telemetry-service/src/handlers/events.ts @@ -2,6 +2,7 @@ import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; import { requiredEnv } from "@srads/common/env"; import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { parseIngestBody } from "../lib/ingest.js"; +import { runEventExists } from "../lib/lookup.js"; const sqs = new SQSClient({}); const queueUrl = requiredEnv("EVENT_QUEUE_URL"); @@ -21,6 +22,10 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { const parsed = parseIngestBody(body); if ("error" in parsed) return jsonResponse(400, { error: parsed.error }); + if (parsed.runId && (await runEventExists(parsed.algorithmId, parsed.runId, parsed.event))) { + return jsonResponse(409, { error: "event already recorded for this run" }); + } + await sqs.send(new SendMessageCommand({ QueueUrl: queueUrl, MessageBody: JSON.stringify(parsed) })); return jsonResponse(202, { ok: true }); }; diff --git a/rads/telemetry-service/src/lib/lookup.ts b/rads/telemetry-service/src/lib/lookup.ts new file mode 100644 index 00000000..d06029c1 --- /dev/null +++ b/rads/telemetry-service/src/lib/lookup.ts @@ -0,0 +1,23 @@ +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb"; +import { requiredEnv } from "@srads/common/env"; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const table = requiredEnv("RUN_LOG_TABLE"); + +export async function runEventExists(algorithmId: string, runId: string, event: string): Promise { + let exclusiveStartKey: Record | undefined; + do { + const result = await ddb.send( + new QueryCommand({ + TableName: table, + KeyConditionExpression: "AlgorithmId = :a AND begins_with(Sk, :s)", + ExpressionAttributeValues: { ":a": algorithmId, ":s": `${runId}#` }, + ExclusiveStartKey: exclusiveStartKey, + }), + ); + if ((result.Items ?? []).some((item) => item.Event === event)) return true; + exclusiveStartKey = result.LastEvaluatedKey; + } while (exclusiveStartKey); + return false; +} diff --git a/secure/log-service/README.md b/secure/log-service/README.md index f9c8c2d5..52ba1d75 100644 --- a/secure/log-service/README.md +++ b/secure/log-service/README.md @@ -8,7 +8,7 @@ Run, build, and debug events live in [`../telemetry-service`](../telemetry-servi The public domain carries the subject-access write and read paths: -- `POST /subject-access` - sce-unit posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after a successful DC app run, using the ids PDM returned for that run. The bearer token is verified, so only an attested enclave can write its own algorithm's trail. +- `POST /subject-access` - sce-unit posts one batch of `{algorithmId, runId, dataSubjectIds, timestamp}` after a successful DC app run, using the ids PDM returned for that run. The bearer token is verified, so only an attested enclave can write its own algorithm's trail. If any subject in the batch already has a row for that run, the handler returns `409`. - `GET /subject-access/{dataSubjectId}` - one data subject's chronological access entries as a key query. - `GET /subject-access` - a cursor-paginated view over all access events for the authority dashboard. `?limit=` caps at 1000 (default 500); pass the returned `nextCursor` as `?cursor=` until it is `null`. Scan order is not chronological. @@ -20,7 +20,7 @@ The `rads` baseline has no verifier, so its log-service accepts subject-access w ## Key model -Partition key `DataSubjectId`, sort key `Sk = ##`. That tuple is unique per item: a data subject's record lives on exactly one PDM page, so it is read once per run by exactly one algorithm. A redelivered ingest overwrites an identical row instead of appending a duplicate. +Partition key `DataSubjectId`, sort key `Sk = ##`. That tuple is unique per item: a data subject's record lives on exactly one PDM page, so it is read once per run by exactly one algorithm. A repeat write for the same `(algorithmId, runId)` is rejected with `409` rather than appended under a new timestamp. ## Prerequisites (manual, one-time, outside this repo's IaC) diff --git a/secure/log-service/src/handlers/subjectAccessIngest.ts b/secure/log-service/src/handlers/subjectAccessIngest.ts index 681646e5..b6e118c4 100644 --- a/secure/log-service/src/handlers/subjectAccessIngest.ts +++ b/secure/log-service/src/handlers/subjectAccessIngest.ts @@ -5,6 +5,7 @@ import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { retryAsync } from "ts-retry"; import { requireEnclaveToken } from "../lib/auth.js"; import { parseSubjectAccessBody, toItem } from "../lib/subjectAccess.js"; +import { hasExistingAccess } from "../lib/subjectAccessDuplicates.js"; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); const table = requiredEnv("SUBJECT_ACCESS_TABLE"); @@ -54,9 +55,14 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { const authError = await requireEnclaveToken(event.headers.authorization, parsed.algorithmId, parsed.runId); if (authError) return jsonResponse(401, { error: authError }); + const dataSubjectIds = [...new Set(parsed.dataSubjectIds)]; + if (await hasExistingAccess(ddb, table, parsed.algorithmId, parsed.runId, dataSubjectIds)) { + return jsonResponse(409, { error: "subject access already recorded for this run" }); + } + // A DS-keyed subject-access row per data subject; the sort key is (timestamp, algorithmId, runId). // BatchWrite takes WriteRequests, not bare items: the document client only marshals PutRequest.Item. - const items = [...new Set(parsed.dataSubjectIds)].map((dataSubjectId) => ({ + const items = dataSubjectIds.map((dataSubjectId) => ({ PutRequest: { Item: toItem({ dataSubjectId, diff --git a/secure/log-service/src/lib/subjectAccessDuplicates.ts b/secure/log-service/src/lib/subjectAccessDuplicates.ts new file mode 100644 index 00000000..d396a014 --- /dev/null +++ b/secure/log-service/src/lib/subjectAccessDuplicates.ts @@ -0,0 +1,41 @@ +import { type DynamoDBDocumentClient, paginateQuery } from "@aws-sdk/lib-dynamodb"; + +const CHUNK = 25; + +async function subjectHasAccess( + ddb: DynamoDBDocumentClient, + table: string, + algorithmId: string, + runId: string, + dataSubjectId: string, +): Promise { + const pages = paginateQuery( + { client: ddb }, + { + TableName: table, + KeyConditionExpression: "DataSubjectId = :s", + FilterExpression: "AlgorithmId = :a AND RunId = :r", + ExpressionAttributeValues: { ":s": dataSubjectId, ":a": algorithmId, ":r": runId }, + }, + ); + for await (const page of pages) { + if ((page.Items ?? []).length > 0) return true; + } + return false; +} + +export async function hasExistingAccess( + ddb: DynamoDBDocumentClient, + table: string, + algorithmId: string, + runId: string, + dataSubjectIds: string[], +): Promise { + for (let i = 0; i < dataSubjectIds.length; i += CHUNK) { + const found = await Promise.all( + dataSubjectIds.slice(i, i + CHUNK).map((id) => subjectHasAccess(ddb, table, algorithmId, runId, id)), + ); + if (found.includes(true)) return true; + } + return false; +} diff --git a/secure/telemetry-service/README.md b/secure/telemetry-service/README.md index 393a5b1b..a1d17229 100644 --- a/secure/telemetry-service/README.md +++ b/secure/telemetry-service/README.md @@ -4,7 +4,7 @@ Serverless Framework v4, TypeScript. Diagnostic timeline for performance, test, ## Endpoints -- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. No token and no client certificate. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. +- `POST /events` - producers post `{algorithmId, runId?, event, timestamp, detail?}`. No token and no client certificate. The event is validated against the allowlist in `src/lib/events.ts`, enqueued, and the handler returns `202`. If that run already recorded the same event, the handler returns `409` instead of enqueuing another row. - `GET /logs/{algorithmId}` - exports one algorithm's timeline. `?runId=` narrows to a single run. Pages through all results. - `GET /logs` - exports the whole table across all algorithms. diff --git a/secure/telemetry-service/src/handlers/events.ts b/secure/telemetry-service/src/handlers/events.ts index b06caf73..c0484950 100644 --- a/secure/telemetry-service/src/handlers/events.ts +++ b/secure/telemetry-service/src/handlers/events.ts @@ -1,5 +1,6 @@ import type { APIGatewayProxyHandlerV2 } from "aws-lambda"; import { parseIngestBody } from "../lib/ingest.js"; +import { runEventExists } from "../lib/lookup.js"; import { enqueueEvent } from "../lib/queue.js"; function jsonResponse(statusCode: number, body: unknown) { @@ -19,6 +20,10 @@ export const handler: APIGatewayProxyHandlerV2 = async (event) => { const parsed = parseIngestBody(body); if ("error" in parsed) return jsonResponse(400, { error: parsed.error }); + if (parsed.runId && (await runEventExists(parsed.algorithmId, parsed.runId, parsed.event))) { + return jsonResponse(409, { error: "event already recorded for this run" }); + } + await enqueueEvent(parsed); return jsonResponse(202, { ok: true }); }; diff --git a/secure/telemetry-service/src/lib/lookup.ts b/secure/telemetry-service/src/lib/lookup.ts new file mode 100644 index 00000000..d06029c1 --- /dev/null +++ b/secure/telemetry-service/src/lib/lookup.ts @@ -0,0 +1,23 @@ +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb"; +import { requiredEnv } from "@srads/common/env"; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const table = requiredEnv("RUN_LOG_TABLE"); + +export async function runEventExists(algorithmId: string, runId: string, event: string): Promise { + let exclusiveStartKey: Record | undefined; + do { + const result = await ddb.send( + new QueryCommand({ + TableName: table, + KeyConditionExpression: "AlgorithmId = :a AND begins_with(Sk, :s)", + ExpressionAttributeValues: { ":a": algorithmId, ":s": `${runId}#` }, + ExclusiveStartKey: exclusiveStartKey, + }), + ); + if ((result.Items ?? []).some((item) => item.Event === event)) return true; + exclusiveStartKey = result.LastEvaluatedKey; + } while (exclusiveStartKey); + return false; +}