From f5803436baef79b3063690213d02c1e8c2f2675a Mon Sep 17 00:00:00 2001 From: Yulong Yang Date: Thu, 19 Nov 2026 00:32:01 +0000 Subject: [PATCH] Regenerate SDK and release 0.1.39 Regenerated with `kapigen --public-api-output-path` (per README) from kaggleazure ci at 48eb12ab73a (48eb12ab73aa9eb50f3b845f6d486c1f77978036), run with `dotnet run` and .NET SDK 10.0.100. Only `kagglesdk/` is copied from the output; `kagglesdk/test/` is kept. `__version__` goes to 0.1.39. Breaking changes: - `ApiValidationStage` moves from `kagglesdk.benchmarks.types.benchmark_tasks_api_service` to `kagglesdk.benchmarks.types.benchmark_types`. Importing it from the old module now raises `ImportError`. `ApiTaskCreationValidationConfig` moves too, but the old module still imports it, so the old import path works. - `BenchmarksApiClient.update_benchmark_arena_combinations` now returns `ApiUpdateBenchmarkArenaCombinationsResponse` instead of `None`. Other changes surfaced by the regen: - Add `get_benchmark_task_collaborators` and `update_benchmark_task_collaborators` to `BenchmarkTasksApiClient`, with `ApiGetBenchmarkTaskCollaboratorsRequest`, `ApiUpdateBenchmarkTaskCollaboratorsRequest`, `ApiBenchmarkTaskCollaborators`, `ApiBenchmarkTaskCollaborator`, `ApiBenchmarkTaskPrincipal` and the `BenchmarkTaskCollaboratorRole` enum. - Add `create_benchmark_metric_producer`, `batch_schedule_benchmark_metrics` and `validate_benchmark_metric_producer_output` to `BenchmarksApiClient`, with their request and response types, `BenchmarkMetricProducer` and `BenchmarkMetricProducerInputOptions`. - Add `HarborTaskEnvironment`, used by the new `BenchmarkTaskKaggleDatasetsDefinition.environment`. - Add fields to existing types: `ApiBenchmarkTask.validation_config`, `preview_artifact_path` and `preview_artifact_type` (also on `ApiUpdateBenchmarkTaskVersionRequest`); `ApiBenchmarkTaskRun.is_manual`; `BenchmarkArenaSettings.show_vote_counts`, `vote_count` and `votes_task_version_id`; `BenchmarkSchedulerSettings.kkb_pool_gating_enabled`, `max_kkb_pool_queued_sessions`, `unattributed_scheduler_priority` and `user_session_caps_enforced`; `dataset_id`, `dataset_version_id` and `databundle_version_id` on `DatasetMount` and `DefinitionSource`. - Docs: `NvidiaTeslaP100` drops out of the kernels `machine_shape` list, and `GetBenchmarkSchedulerStatusResponse.model_version_stats` documents its new ordering. No files added or removed. Apart from the two moves above, no type, method, field or enum value is removed, and no other signature changes. --- kagglesdk/__init__.py | 2 +- .../services/benchmark_tasks_api_service.py | 32 +- .../services/benchmarks_api_service.py | 89 +- kagglesdk/benchmarks/types/benchmark_enums.py | 10 + .../types/benchmark_task_run_service.py | 6 +- .../types/benchmark_tasks_api_service.py | 572 ++++++++---- kagglesdk/benchmarks/types/benchmark_types.py | 835 +++++++++++++++++- .../types/benchmarks_api_service.py | 526 ++++++++++- .../kernels/types/kernels_api_service.py | 4 - 9 files changed, 1907 insertions(+), 169 deletions(-) diff --git a/kagglesdk/__init__.py b/kagglesdk/__init__.py index 1a487f5..d1b1fb7 100644 --- a/kagglesdk/__init__.py +++ b/kagglesdk/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.38" +__version__ = "0.1.39" from kagglesdk.kaggle_client import KaggleClient from kagglesdk.kaggle_creds import KaggleCredentials diff --git a/kagglesdk/benchmarks/services/benchmark_tasks_api_service.py b/kagglesdk/benchmarks/services/benchmark_tasks_api_service.py index c8cc245..1599cc2 100644 --- a/kagglesdk/benchmarks/services/benchmark_tasks_api_service.py +++ b/kagglesdk/benchmarks/services/benchmark_tasks_api_service.py @@ -1,5 +1,5 @@ from kagglesdk.benchmarks.types.benchmark_task_run_service import GetBenchmarkSchedulerStatusResponse -from kagglesdk.benchmarks.types.benchmark_tasks_api_service import ApiBatchScheduleBenchmarkTaskRunsRequest, ApiBatchScheduleBenchmarkTaskRunsResponse, ApiBenchmarkTask, ApiCancelBenchmarkTaskRunsRequest, ApiCancelBenchmarkTaskRunsResponse, ApiCreateBenchmarkTaskRequest, ApiDownloadBenchmarkTaskRunOutputRequest, ApiGetBenchmarkSchedulerStatusRequest, ApiGetBenchmarkTaskQuotaRequest, ApiGetBenchmarkTaskQuotaResponse, ApiGetBenchmarkTaskRequest, ApiGetBenchmarkTaskRunLogsRequest, ApiListBenchmarkTaskRunsRequest, ApiListBenchmarkTaskRunsResponse, ApiListBenchmarkTasksRequest, ApiListBenchmarkTasksResponse, ApiPublishBenchmarkTaskRequest, ApiUpdateBenchmarkTaskRequest, ApiUpdateBenchmarkTaskResponse, ApiUpdateBenchmarkTaskVersionRequest +from kagglesdk.benchmarks.types.benchmark_tasks_api_service import ApiBatchScheduleBenchmarkTaskRunsRequest, ApiBatchScheduleBenchmarkTaskRunsResponse, ApiBenchmarkTask, ApiBenchmarkTaskCollaborators, ApiCancelBenchmarkTaskRunsRequest, ApiCancelBenchmarkTaskRunsResponse, ApiCreateBenchmarkTaskRequest, ApiDownloadBenchmarkTaskRunOutputRequest, ApiGetBenchmarkSchedulerStatusRequest, ApiGetBenchmarkTaskCollaboratorsRequest, ApiGetBenchmarkTaskQuotaRequest, ApiGetBenchmarkTaskQuotaResponse, ApiGetBenchmarkTaskRequest, ApiGetBenchmarkTaskRunLogsRequest, ApiListBenchmarkTaskRunsRequest, ApiListBenchmarkTaskRunsResponse, ApiListBenchmarkTasksRequest, ApiListBenchmarkTasksResponse, ApiPublishBenchmarkTaskRequest, ApiUpdateBenchmarkTaskCollaboratorsRequest, ApiUpdateBenchmarkTaskRequest, ApiUpdateBenchmarkTaskResponse, ApiUpdateBenchmarkTaskVersionRequest from kagglesdk.common.types.file_download import FileDownload from kagglesdk.kaggle_http_client import KaggleHttpClient @@ -208,3 +208,33 @@ def update_benchmark_task_version(self, request: ApiUpdateBenchmarkTaskVersionRe request = ApiUpdateBenchmarkTaskVersionRequest() return self._client.call("benchmarks.BenchmarkTasksApiService", "UpdateBenchmarkTaskVersion", request, ApiBenchmarkTask) + + def get_benchmark_task_collaborators(self, request: ApiGetBenchmarkTaskCollaboratorsRequest = None) -> ApiBenchmarkTaskCollaborators: + r""" + List the users and groups who collaborate on a task and the role each one + holds. The owner isn't included. + + Args: + request (ApiGetBenchmarkTaskCollaboratorsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiGetBenchmarkTaskCollaboratorsRequest() + + return self._client.call("benchmarks.BenchmarkTasksApiService", "GetBenchmarkTaskCollaborators", request, ApiBenchmarkTaskCollaborators) + + def update_benchmark_task_collaborators(self, request: ApiUpdateBenchmarkTaskCollaboratorsRequest = None) -> ApiBenchmarkTaskCollaborators: + r""" + Add, re-role or remove collaborators on a task. Collaborators the request + doesn't name are left as they are. + + Args: + request (ApiUpdateBenchmarkTaskCollaboratorsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiUpdateBenchmarkTaskCollaboratorsRequest() + + return self._client.call("benchmarks.BenchmarkTasksApiService", "UpdateBenchmarkTaskCollaborators", request, ApiBenchmarkTaskCollaborators) diff --git a/kagglesdk/benchmarks/services/benchmarks_api_service.py b/kagglesdk/benchmarks/services/benchmarks_api_service.py index 040215f..6314246 100644 --- a/kagglesdk/benchmarks/services/benchmarks_api_service.py +++ b/kagglesdk/benchmarks/services/benchmarks_api_service.py @@ -1,5 +1,5 @@ -from kagglesdk.benchmarks.types.benchmark_types import BenchmarkArenaSettings -from kagglesdk.benchmarks.types.benchmarks_api_service import ApiAddTasksToBenchmarkRequest, ApiAddTasksToBenchmarkResponse, ApiBenchmarkLeaderboard, ApiBenchmarkModelVersionConfig, ApiCreateBenchmarkArenaSettingsRequest, ApiCreateBenchmarkModelVersionConfigRequest, ApiCreateBenchmarkVersionAgentMappingsRequest, ApiCreateBenchmarkVersionAgentMappingsResponse, ApiDeleteBenchmarkVersionAgentMappingsRequest, ApiDeleteBenchmarkVersionAgentMappingsResponse, ApiGetBenchmarkArenaSettingsRequest, ApiGetBenchmarkLeaderboardRequest, ApiGetBenchmarkModelVersionConfigRequest, ApiListBenchmarkArenaCombinationsRequest, ApiListBenchmarkArenaCombinationsResponse, ApiListBenchmarkModelsRequest, ApiListBenchmarkModelsResponse, ApiListBenchmarkModelVersionConfigsRequest, ApiListBenchmarkModelVersionConfigsResponse, ApiListBenchmarkVersionAgentMappingsRequest, ApiListBenchmarkVersionAgentMappingsResponse, ApiRegisterAuxiliaryModelVersionConfigsRequest, ApiRemoveTasksFromBenchmarkRequest, ApiRemoveTasksFromBenchmarkResponse, ApiUnregisterAuxiliaryModelVersionConfigsRequest, ApiUpdateBenchmarkArenaCombinationsRequest, ApiUpdateBenchmarkArenaSettingsRequest +from kagglesdk.benchmarks.types.benchmark_types import BenchmarkArenaSettings, BenchmarkMetricProducer +from kagglesdk.benchmarks.types.benchmarks_api_service import ApiAddTasksToBenchmarkRequest, ApiAddTasksToBenchmarkResponse, ApiBatchScheduleBenchmarkMetricsRequest, ApiBatchScheduleBenchmarkMetricsResponse, ApiBenchmarkLeaderboard, ApiBenchmarkModelVersionConfig, ApiCreateBenchmarkArenaSettingsRequest, ApiCreateBenchmarkMetricProducerRequest, ApiCreateBenchmarkModelVersionConfigRequest, ApiCreateBenchmarkVersionAgentMappingsRequest, ApiCreateBenchmarkVersionAgentMappingsResponse, ApiDeleteBenchmarkVersionAgentMappingsRequest, ApiDeleteBenchmarkVersionAgentMappingsResponse, ApiGetBenchmarkArenaSettingsRequest, ApiGetBenchmarkLeaderboardRequest, ApiGetBenchmarkModelVersionConfigRequest, ApiListBenchmarkArenaCombinationsRequest, ApiListBenchmarkArenaCombinationsResponse, ApiListBenchmarkModelsRequest, ApiListBenchmarkModelsResponse, ApiListBenchmarkModelVersionConfigsRequest, ApiListBenchmarkModelVersionConfigsResponse, ApiListBenchmarkVersionAgentMappingsRequest, ApiListBenchmarkVersionAgentMappingsResponse, ApiRegisterAuxiliaryModelVersionConfigsRequest, ApiRemoveTasksFromBenchmarkRequest, ApiRemoveTasksFromBenchmarkResponse, ApiUnregisterAuxiliaryModelVersionConfigsRequest, ApiUpdateBenchmarkArenaCombinationsRequest, ApiUpdateBenchmarkArenaCombinationsResponse, ApiUpdateBenchmarkArenaSettingsRequest, ApiValidateBenchmarkMetricProducerOutputRequest, ApiValidateBenchmarkMetricProducerOutputResponse from kagglesdk.kaggle_http_client import KaggleHttpClient class BenchmarksApiClient(object): @@ -376,11 +376,12 @@ def list_benchmark_arena_combinations(self, request: ApiListBenchmarkArenaCombin return self._client.call("benchmarks.BenchmarksApiService", "ListBenchmarkArenaCombinations", request, ApiListBenchmarkArenaCombinationsResponse) - def update_benchmark_arena_combinations(self, request: ApiUpdateBenchmarkArenaCombinationsRequest = None): + def update_benchmark_arena_combinations(self, request: ApiUpdateBenchmarkArenaCombinationsRequest = None) -> ApiUpdateBenchmarkArenaCombinationsResponse: r""" Opt (task version, candidate) combinations into or out of a benchmark version's arena. Requires update access to the benchmark version and read - access to each candidate added. + access to each candidate added. Added combinations with nothing to show + voters are skipped and returned; the rest are saved. Example: curl -sSL -u fickleone:local_api_token \ @@ -402,4 +403,82 @@ def update_benchmark_arena_combinations(self, request: ApiUpdateBenchmarkArenaCo if request is None: request = ApiUpdateBenchmarkArenaCombinationsRequest() - self._client.call("benchmarks.BenchmarksApiService", "UpdateBenchmarkArenaCombinations", request, None) + return self._client.call("benchmarks.BenchmarksApiService", "UpdateBenchmarkArenaCombinations", request, ApiUpdateBenchmarkArenaCombinationsResponse) + + def create_benchmark_metric_producer(self, request: ApiCreateBenchmarkMetricProducerRequest = None) -> BenchmarkMetricProducer: + r""" + Create a BenchmarkMetricProducer on a benchmark version. Requires update + access to the benchmark version and read access to the definition and + mounted datasets. + + Example: + curl -sSL -u fickleone:local_api_token \ + -X POST \ + http://localhost/api/v1/benchmarks/fickleone/my-benchmark/metric-producers/create + \ + -H 'Content-Type: application/json' \ + -d '{ + 'definitionSource': { + 'datasetVersionSlug': 'fickleone/my-metrics/versions/1' + }, + 'entrypoint': 'main.py', + 'inputOptions': {'resultsOnly': true} + }' + + Args: + request (ApiCreateBenchmarkMetricProducerRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiCreateBenchmarkMetricProducerRequest() + + return self._client.call("benchmarks.BenchmarksApiService", "CreateBenchmarkMetricProducer", request, BenchmarkMetricProducer) + + def batch_schedule_benchmark_metrics(self, request: ApiBatchScheduleBenchmarkMetricsRequest = None) -> ApiBatchScheduleBenchmarkMetricsResponse: + r""" + Run one of a benchmark version's metric producers: stages the version's + latest completed runs, then starts the producer's session. Requires update + access to the benchmark version. + + Example: + curl -sSL -u fickleone:local_api_token \ + -X POST \ + http://localhost/api/v1/benchmarks/fickleone/my-benchmark/metrics/schedule + \ + -H 'Content-Type: application/json' \ + -d '{'benchmarkMetricProducerId': 1}' + + Args: + request (ApiBatchScheduleBenchmarkMetricsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiBatchScheduleBenchmarkMetricsRequest() + + return self._client.call("benchmarks.BenchmarksApiService", "BatchScheduleBenchmarkMetrics", request, ApiBatchScheduleBenchmarkMetricsResponse) + + def validate_benchmark_metric_producer_output(self, request: ApiValidateBenchmarkMetricProducerOutputRequest = None) -> ApiValidateBenchmarkMetricProducerOutputResponse: + r""" + Validate a BenchmarkMetricProducerOutput JSON payload against a benchmark + version's mapped candidates and manual metrics without persisting anything. + Requires update access to the benchmark version. + + Example: + curl -sSL -u fickleone:local_api_token \ + -X POST \ + http://localhost/api/v1/benchmarks/fickleone/my-benchmark/metric-producers/validate-output + \ + -H 'Content-Type: application/json' \ + -d '{'outputJson': '{\'version\':1,\'metrics\':[...]}'}' + + Args: + request (ApiValidateBenchmarkMetricProducerOutputRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiValidateBenchmarkMetricProducerOutputRequest() + + return self._client.call("benchmarks.BenchmarksApiService", "ValidateBenchmarkMetricProducerOutput", request, ApiValidateBenchmarkMetricProducerOutputResponse) diff --git a/kagglesdk/benchmarks/types/benchmark_enums.py b/kagglesdk/benchmarks/types/benchmark_enums.py index a588c2f..fcbc166 100644 --- a/kagglesdk/benchmarks/types/benchmark_enums.py +++ b/kagglesdk/benchmarks/types/benchmark_enums.py @@ -196,6 +196,16 @@ class BenchmarkLeaderboardDisplayType(enum.Enum): $2,601,845, $456, $790 """ +class BenchmarkTaskCollaboratorRole(enum.Enum): + """What a task collaborator can do. Each role includes the ones above it.""" + BENCHMARK_TASK_COLLABORATOR_ROLE_UNSPECIFIED = 0 + BENCHMARK_TASK_COLLABORATOR_ROLE_VIEWER = 1 + """Can view the task while it's private.""" + BENCHMARK_TASK_COLLABORATOR_ROLE_EDITOR = 2 + """Can also edit the task.""" + BENCHMARK_TASK_COLLABORATOR_ROLE_ADMIN = 3 + """Can also change the task's visibility and collaborators.""" + class BenchmarkTaskRunState(enum.Enum): BENCHMARK_TASK_RUN_STATE_UNSPECIFIED = 0 BENCHMARK_TASK_RUN_STATE_QUEUED = 1 diff --git a/kagglesdk/benchmarks/types/benchmark_task_run_service.py b/kagglesdk/benchmarks/types/benchmark_task_run_service.py index 35ce498..6fa6f57 100644 --- a/kagglesdk/benchmarks/types/benchmark_task_run_service.py +++ b/kagglesdk/benchmarks/types/benchmark_task_run_service.py @@ -454,7 +454,8 @@ class GetBenchmarkSchedulerStatusResponse(KaggleObject): Excludes cancellations, which are stamped Errored and removed. model_version_stats (BenchmarkSchedulerModelVersionStats) One entry per model version with live work or a recent outcome, ordered - by running count descending, then id ascending so ties stay stable. + by queued count descending, then running count descending, then id + ascending so ties stay stable. settings (BenchmarkSchedulerSettings) The settings these counts were measured against. The caps are enforced over the enqueued-only population, so headroom is their difference from the @@ -515,7 +516,8 @@ def recent_state_counts(self, recent_state_counts: Optional[List[Optional['Bench def model_version_stats(self) -> Optional[List[Optional['BenchmarkSchedulerModelVersionStats']]]: r""" One entry per model version with live work or a recent outcome, ordered - by running count descending, then id ascending so ties stay stable. + by queued count descending, then running count descending, then id + ascending so ties stay stable. """ return self._model_version_stats diff --git a/kagglesdk/benchmarks/types/benchmark_tasks_api_service.py b/kagglesdk/benchmarks/types/benchmark_tasks_api_service.py index 74338be..6de4c11 100644 --- a/kagglesdk/benchmarks/types/benchmark_tasks_api_service.py +++ b/kagglesdk/benchmarks/types/benchmark_tasks_api_service.py @@ -1,8 +1,8 @@ from datetime import datetime, timedelta from google.protobuf.field_mask_pb2 import FieldMask -from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkArenaArtifactType, BenchmarkCandidateType, BenchmarkEnqueueSkipPolicy, BenchmarkTaskRunState, BenchmarkTaskVersionCreationState, BenchmarkTaskVersionSource +from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkArenaArtifactType, BenchmarkCandidateType, BenchmarkEnqueueSkipPolicy, BenchmarkTaskCollaboratorRole, BenchmarkTaskRunState, BenchmarkTaskVersionCreationState, BenchmarkTaskVersionSource from kagglesdk.benchmarks.types.benchmark_task_run_service import BatchScheduleBenchmarkModelVersionResult -from kagglesdk.benchmarks.types.benchmark_types import BenchmarkTaskDefinition, BenchmarkTaskOptions, BenchmarkTaskValidationStage +from kagglesdk.benchmarks.types.benchmark_types import ApiTaskCreationValidationConfig, BenchmarkTaskDefinition, BenchmarkTaskOptions from kagglesdk.kaggle_object import * from typing import List, Optional @@ -383,8 +383,13 @@ class ApiBenchmarkTask(KaggleObject): reader cannot read are omitted, and `env_variables` is omitted wholesale unless the caller could update the task. `definition_partial` is set whenever anything is omitted for one of those reasons. + + A client pushing this back should not also send `validation_config` with + a default_*_slug set here; see `validation_config`. definition_partial (bool) - True when `definition` is missing something the version declares. + True when `definition` or `validation_config` omits an item the version + declares. A `validation_config` left unset for a caller who may not read + it at all does not count; see that field. Separates 'declares no default candidate' from 'declares one that did not come back', which otherwise look identical and would have a client push a spurious change. False when `definition` is unset. @@ -393,14 +398,32 @@ class ApiBenchmarkTask(KaggleObject): when a referenced dataset cannot be named -- one still being created, say -- so a client cannot read it as a statement about its own permissions. artifact_path (str) - Glob, relative to a run's output root, naming the file the task's preview - shows, e.g. *.mp4. Unset if the task has none. + Glob, relative to a run's output root, naming the file a benchmark's arena + and gallery show, e.g. *.mp4. Unset if the task has none. artifact_type (BenchmarkArenaArtifactType) How the artifact is rendered. Set iff artifact_path is. are_run_artifacts_public (bool) Whether everyone who can read the task can also read its raw run artifacts (run.json, logs, output files, trajectories), rather than only the task's writers. + preview_artifact_path (str) + preview_artifact_type (BenchmarkArenaArtifactType) + validation_config (ApiTaskCreationValidationConfig) + The validation stages this task version declared, returned in the shape + CreateBenchmarkTask accepts, with the version's default candidate on the + DEFAULT_CANDIDATE stage. Populated alongside `definition`, only when + `include_definition` is set. Unset when the version recorded no + validation stages, as tasks created before they existed did. + + The default candidate also still comes back on `definition` as its legacy + default_*_slug field. A client pushing a task back should send either + `validation_config` or the legacy definition default_*_slug, not both, + because CreateBenchmarkTask rejects a request that sets both. + + Only a caller who could update the task, or a site admin, reads the + stages; for anyone else this is always unset, and that alone does not + set `definition_partial`. A default candidate the reader cannot read is + omitted from its stage, and `definition_partial` is set. """ def __init__(self): @@ -425,6 +448,9 @@ def __init__(self): self._artifact_path = None self._artifact_type = None self._are_run_artifacts_public = None + self._preview_artifact_path = None + self._preview_artifact_type = None + self._validation_config = None self._freeze() @property @@ -724,6 +750,9 @@ def definition(self) -> Optional['BenchmarkTaskDefinition']: reader cannot read are omitted, and `env_variables` is omitted wholesale unless the caller could update the task. `definition_partial` is set whenever anything is omitted for one of those reasons. + + A client pushing this back should not also send `validation_config` with + a default_*_slug set here; see `validation_config`. """ return self._definition or None @@ -739,7 +768,9 @@ def definition(self, definition: Optional[Optional['BenchmarkTaskDefinition']]): @property def definition_partial(self) -> bool: r""" - True when `definition` is missing something the version declares. + True when `definition` or `validation_config` omits an item the version + declares. A `validation_config` left unset for a caller who may not read + it at all does not count; see that field. Separates 'declares no default candidate' from 'declares one that did not come back', which otherwise look identical and would have a client push a spurious change. False when `definition` is unset. @@ -762,8 +793,8 @@ def definition_partial(self, definition_partial: bool): @property def artifact_path(self) -> str: r""" - Glob, relative to a run's output root, naming the file the task's preview - shows, e.g. *.mp4. Unset if the task has none. + Glob, relative to a run's output root, naming the file a benchmark's arena + and gallery show, e.g. *.mp4. Unset if the task has none. """ return self._artifact_path or "" @@ -790,6 +821,62 @@ def artifact_type(self, artifact_type: Optional['BenchmarkArenaArtifactType']): raise TypeError('artifact_type must be of type BenchmarkArenaArtifactType') self._artifact_type = artifact_type + @property + def preview_artifact_path(self) -> str: + return self._preview_artifact_path or "" + + @preview_artifact_path.setter + def preview_artifact_path(self, preview_artifact_path: Optional[str]): + if preview_artifact_path is None: + del self.preview_artifact_path + return + if not isinstance(preview_artifact_path, str): + raise TypeError('preview_artifact_path must be of type str') + self._preview_artifact_path = preview_artifact_path + + @property + def preview_artifact_type(self) -> 'BenchmarkArenaArtifactType': + return self._preview_artifact_type or BenchmarkArenaArtifactType.BENCHMARK_ARENA_ARTIFACT_TYPE_UNSPECIFIED + + @preview_artifact_type.setter + def preview_artifact_type(self, preview_artifact_type: Optional['BenchmarkArenaArtifactType']): + if preview_artifact_type is None: + del self.preview_artifact_type + return + if not isinstance(preview_artifact_type, BenchmarkArenaArtifactType): + raise TypeError('preview_artifact_type must be of type BenchmarkArenaArtifactType') + self._preview_artifact_type = preview_artifact_type + + @property + def validation_config(self) -> Optional['ApiTaskCreationValidationConfig']: + r""" + The validation stages this task version declared, returned in the shape + CreateBenchmarkTask accepts, with the version's default candidate on the + DEFAULT_CANDIDATE stage. Populated alongside `definition`, only when + `include_definition` is set. Unset when the version recorded no + validation stages, as tasks created before they existed did. + + The default candidate also still comes back on `definition` as its legacy + default_*_slug field. A client pushing a task back should send either + `validation_config` or the legacy definition default_*_slug, not both, + because CreateBenchmarkTask rejects a request that sets both. + + Only a caller who could update the task, or a site admin, reads the + stages; for anyone else this is always unset, and that alone does not + set `definition_partial`. A default candidate the reader cannot read is + omitted from its stage, and `definition_partial` is set. + """ + return self._validation_config or None + + @validation_config.setter + def validation_config(self, validation_config: Optional[Optional['ApiTaskCreationValidationConfig']]): + if validation_config is None: + del self.validation_config + return + if not isinstance(validation_config, ApiTaskCreationValidationConfig): + raise TypeError('validation_config must be of type ApiTaskCreationValidationConfig') + self._validation_config = validation_config + class ApiBenchmarkTaskCitation(KaggleObject): r""" @@ -840,6 +927,122 @@ def url(self, url: str): self._url = url +class ApiBenchmarkTaskCollaborator(KaggleObject): + r""" + Attributes: + principal (ApiBenchmarkTaskPrincipal) + role (BenchmarkTaskCollaboratorRole) + """ + + def __init__(self): + self._principal = None + self._role = BenchmarkTaskCollaboratorRole.BENCHMARK_TASK_COLLABORATOR_ROLE_UNSPECIFIED + self._freeze() + + @property + def principal(self) -> Optional['ApiBenchmarkTaskPrincipal']: + return self._principal + + @principal.setter + def principal(self, principal: Optional['ApiBenchmarkTaskPrincipal']): + if principal is None: + del self.principal + return + if not isinstance(principal, ApiBenchmarkTaskPrincipal): + raise TypeError('principal must be of type ApiBenchmarkTaskPrincipal') + self._principal = principal + + @property + def role(self) -> 'BenchmarkTaskCollaboratorRole': + return self._role + + @role.setter + def role(self, role: 'BenchmarkTaskCollaboratorRole'): + if role is None: + del self.role + return + if not isinstance(role, BenchmarkTaskCollaboratorRole): + raise TypeError('role must be of type BenchmarkTaskCollaboratorRole') + self._role = role + + +class ApiBenchmarkTaskCollaborators(KaggleObject): + r""" + Attributes: + collaborators (ApiBenchmarkTaskCollaborator) + """ + + def __init__(self): + self._collaborators = [] + self._freeze() + + @property + def collaborators(self) -> Optional[List[Optional['ApiBenchmarkTaskCollaborator']]]: + return self._collaborators + + @collaborators.setter + def collaborators(self, collaborators: Optional[List[Optional['ApiBenchmarkTaskCollaborator']]]): + if collaborators is None: + del self.collaborators + return + if not isinstance(collaborators, list): + raise TypeError('collaborators must be of type list') + if not all([isinstance(t, ApiBenchmarkTaskCollaborator) for t in collaborators]): + raise TypeError('collaborators must contain only items of type ApiBenchmarkTaskCollaborator') + self._collaborators = collaborators + + +class ApiBenchmarkTaskPrincipal(KaggleObject): + r""" + A user or group that can collaborate on a task. + + Attributes: + username (str) + group_slug (str) + A user-managed group's slug, or a synchronized (e.g. Google) group's + name. Synchronized groups the caller can't see are listed as + `private-group`. + """ + + def __init__(self): + self._username = None + self._group_slug = None + self._freeze() + + @property + def username(self) -> str: + return self._username or "" + + @username.setter + def username(self, username: str): + if username is None: + del self.username + return + if not isinstance(username, str): + raise TypeError('username must be of type str') + del self.group_slug + self._username = username + + @property + def group_slug(self) -> str: + r""" + A user-managed group's slug, or a synchronized (e.g. Google) group's + name. Synchronized groups the caller can't see are listed as + `private-group`. + """ + return self._group_slug or "" + + @group_slug.setter + def group_slug(self, group_slug: str): + if group_slug is None: + del self.group_slug + return + if not isinstance(group_slug, str): + raise TypeError('group_slug must be of type str') + del self.username + self._group_slug = group_slug + + class ApiBenchmarkTaskRun(KaggleObject): r""" API equivalent of the BenchmarkTaskRun from benchmark_types.proto. @@ -875,6 +1078,10 @@ class ApiBenchmarkTaskRun(KaggleObject): Result the run produced (score or pass/fail). Unset for runs that have not produced a result (e.g. still queued/running, errored, or no result recorded). + is_manual (bool) + Whether the run's result was recorded rather than produced by executing the + task on Kaggle, e.g. uploaded through CreateBenchmarkTaskRuns. True exactly + when BenchmarkTaskRun.manual_user_id is set. """ def __init__(self): @@ -888,6 +1095,7 @@ def __init__(self): self._agent_slug = None self._model_version_config_slug = None self._result = None + self._is_manual = False self._freeze() @property @@ -1048,6 +1256,24 @@ def result(self, result: Optional[Optional['ApiBenchmarkTaskRunResult']]): raise TypeError('result must be of type ApiBenchmarkTaskRunResult') self._result = result + @property + def is_manual(self) -> bool: + r""" + Whether the run's result was recorded rather than produced by executing the + task on Kaggle, e.g. uploaded through CreateBenchmarkTaskRuns. True exactly + when BenchmarkTaskRun.manual_user_id is set. + """ + return self._is_manual + + @is_manual.setter + def is_manual(self, is_manual: bool): + if is_manual is None: + del self.is_manual + return + if not isinstance(is_manual, bool): + raise TypeError('is_manual must be of type bool') + self._is_manual = is_manual + class ApiBenchmarkTaskRunIds(KaggleObject): r""" @@ -1600,6 +1826,46 @@ def endpoint(self): return path.format_map(self.to_field_map(self)) +class ApiGetBenchmarkTaskCollaboratorsRequest(KaggleObject): + r""" + Attributes: + slug (ApiBenchmarkTaskSlug) + The task to read. `owner_slug` is optional and defaults to the + authenticated user. `version_number` is rejected: collaborators belong to + the task, not to a version. + """ + + def __init__(self): + self._slug = None + self._freeze() + + @property + def slug(self) -> Optional['ApiBenchmarkTaskSlug']: + r""" + The task to read. `owner_slug` is optional and defaults to the + authenticated user. `version_number` is rejected: collaborators belong to + the task, not to a version. + """ + return self._slug + + @slug.setter + def slug(self, slug: Optional['ApiBenchmarkTaskSlug']): + if slug is None: + del self.slug + return + if not isinstance(slug, ApiBenchmarkTaskSlug): + raise TypeError('slug must be of type ApiBenchmarkTaskSlug') + self._slug = slug + + def endpoint(self): + path = '/api/v1/benchmarks/tasks/{slug.owner_slug}/{slug.task_slug}/collaborators' + return path.format_map(self.to_field_map(self)) + + @staticmethod + def endpoint_path(): + return '/api/v1/benchmarks/tasks/{slug.owner_slug}/{slug.task_slug}/collaborators' + + class ApiGetBenchmarkTaskQuotaRequest(KaggleObject): r""" """ @@ -1666,8 +1932,8 @@ class ApiGetBenchmarkTaskRequest(KaggleObject): Attributes: slug (ApiBenchmarkTaskSlug) include_definition (bool) - Populate `definition` and `definition_partial`. Off by default since it - costs extra reads. + Populate `definition`, `validation_config` and `definition_partial`. Off + by default since it costs extra reads. """ def __init__(self): @@ -1691,8 +1957,8 @@ def slug(self, slug: Optional['ApiBenchmarkTaskSlug']): @property def include_definition(self) -> bool: r""" - Populate `definition` and `definition_partial`. Off by default since it - costs extra reads. + Populate `definition`, `validation_config` and `definition_partial`. Off + by default since it costs extra reads. """ return self._include_definition @@ -1934,8 +2200,8 @@ class ApiListBenchmarkTasksRequest(KaggleObject): Optional benchmark version to target when benchmark_slug is set. When omitted, the benchmark's current published (or draft) version is used. include_definition (bool) - Populate `definition` and `definition_partial` on each task. Off by - default since it costs extra reads. + Populate `definition`, `validation_config` and `definition_partial` on + each task. Off by default since it costs extra reads. """ def __init__(self): @@ -2075,8 +2341,8 @@ def version_number(self, version_number: Optional[int]): @property def include_definition(self) -> bool: r""" - Populate `definition` and `definition_partial` on each task. Off by - default since it costs extra reads. + Populate `definition`, `validation_config` and `definition_partial` on + each task. Off by default since it costs extra reads. """ return self._include_definition @@ -2296,34 +2562,91 @@ def body_fields(): return '*' -class ApiTaskCreationValidationConfig(KaggleObject): +class ApiUpdateBenchmarkTaskCollaboratorsRequest(KaggleObject): r""" - The public API input shape of TaskCreationValidationConfig. - Attributes: - validation_stages (ApiValidationStage) - The validation stages to run against the task version. + slug (ApiBenchmarkTaskSlug) + The task to update. `owner_slug` is optional and defaults to the + authenticated user. `version_number` is rejected: collaborators belong to + the task, not to a version. + add (ApiBenchmarkTaskCollaborator) + Each is given its role, replacing any role it already holds. + remove (ApiBenchmarkTaskPrincipal) + Each loses whatever role it holds. Ones that aren't collaborators are + skipped. """ def __init__(self): - self._validation_stages = [] + self._slug = None + self._add = [] + self._remove = [] self._freeze() @property - def validation_stages(self) -> Optional[List[Optional['ApiValidationStage']]]: - """The validation stages to run against the task version.""" - return self._validation_stages + def slug(self) -> Optional['ApiBenchmarkTaskSlug']: + r""" + The task to update. `owner_slug` is optional and defaults to the + authenticated user. `version_number` is rejected: collaborators belong to + the task, not to a version. + """ + return self._slug + + @slug.setter + def slug(self, slug: Optional['ApiBenchmarkTaskSlug']): + if slug is None: + del self.slug + return + if not isinstance(slug, ApiBenchmarkTaskSlug): + raise TypeError('slug must be of type ApiBenchmarkTaskSlug') + self._slug = slug + + @property + def add(self) -> Optional[List[Optional['ApiBenchmarkTaskCollaborator']]]: + """Each is given its role, replacing any role it already holds.""" + return self._add - @validation_stages.setter - def validation_stages(self, validation_stages: Optional[List[Optional['ApiValidationStage']]]): - if validation_stages is None: - del self.validation_stages + @add.setter + def add(self, add: Optional[List[Optional['ApiBenchmarkTaskCollaborator']]]): + if add is None: + del self.add return - if not isinstance(validation_stages, list): - raise TypeError('validation_stages must be of type list') - if not all([isinstance(t, ApiValidationStage) for t in validation_stages]): - raise TypeError('validation_stages must contain only items of type ApiValidationStage') - self._validation_stages = validation_stages + if not isinstance(add, list): + raise TypeError('add must be of type list') + if not all([isinstance(t, ApiBenchmarkTaskCollaborator) for t in add]): + raise TypeError('add must contain only items of type ApiBenchmarkTaskCollaborator') + self._add = add + + @property + def remove(self) -> Optional[List[Optional['ApiBenchmarkTaskPrincipal']]]: + r""" + Each loses whatever role it holds. Ones that aren't collaborators are + skipped. + """ + return self._remove + + @remove.setter + def remove(self, remove: Optional[List[Optional['ApiBenchmarkTaskPrincipal']]]): + if remove is None: + del self.remove + return + if not isinstance(remove, list): + raise TypeError('remove must be of type list') + if not all([isinstance(t, ApiBenchmarkTaskPrincipal) for t in remove]): + raise TypeError('remove must contain only items of type ApiBenchmarkTaskPrincipal') + self._remove = remove + + def endpoint(self): + path = '/api/v1/benchmarks/tasks/{slug.owner_slug}/{slug.task_slug}/collaborators/update' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' class ApiUpdateBenchmarkTaskRequest(KaggleObject): @@ -2560,10 +2883,13 @@ class ApiUpdateBenchmarkTaskVersionRequest(KaggleObject): Which of the above to write. Only fields named here are read. `tags` is not writable here: tags are task-scoped, so use UpdateBenchmarkTask. artifact_path (str) - The file the task's preview shows, as a glob relative to a run's output - root. Written together with artifact_type: naming either in update_mask - writes both, and leaving both unset clears them. + The file a benchmark's arena and gallery show, as a glob relative to a + run's output root. Written together with artifact_type: naming either in + update_mask writes both, and leaving both unset clears them. artifact_type (BenchmarkArenaArtifactType) + preview_artifact_path (str) + Optional preview shown on cards and while HTML loads; paired with type. + preview_artifact_type (BenchmarkArenaArtifactType) """ def __init__(self): @@ -2574,6 +2900,8 @@ def __init__(self): self._update_mask = None self._artifact_path = None self._artifact_type = None + self._preview_artifact_path = None + self._preview_artifact_type = None self._freeze() @property @@ -2651,9 +2979,9 @@ def citations(self, citations: Optional[List[Optional['ApiBenchmarkTaskCitation' @property def artifact_path(self) -> str: r""" - The file the task's preview shows, as a glob relative to a run's output - root. Written together with artifact_type: naming either in update_mask - writes both, and leaving both unset clears them. + The file a benchmark's arena and gallery show, as a glob relative to a + run's output root. Written together with artifact_type: naming either in + update_mask writes both, and leaving both unset clears them. """ return self._artifact_path or "" @@ -2679,6 +3007,33 @@ def artifact_type(self, artifact_type: Optional['BenchmarkArenaArtifactType']): raise TypeError('artifact_type must be of type BenchmarkArenaArtifactType') self._artifact_type = artifact_type + @property + def preview_artifact_path(self) -> str: + """Optional preview shown on cards and while HTML loads; paired with type.""" + return self._preview_artifact_path or "" + + @preview_artifact_path.setter + def preview_artifact_path(self, preview_artifact_path: Optional[str]): + if preview_artifact_path is None: + del self.preview_artifact_path + return + if not isinstance(preview_artifact_path, str): + raise TypeError('preview_artifact_path must be of type str') + self._preview_artifact_path = preview_artifact_path + + @property + def preview_artifact_type(self) -> 'BenchmarkArenaArtifactType': + return self._preview_artifact_type or BenchmarkArenaArtifactType.BENCHMARK_ARENA_ARTIFACT_TYPE_UNSPECIFIED + + @preview_artifact_type.setter + def preview_artifact_type(self, preview_artifact_type: Optional['BenchmarkArenaArtifactType']): + if preview_artifact_type is None: + del self.preview_artifact_type + return + if not isinstance(preview_artifact_type, BenchmarkArenaArtifactType): + raise TypeError('preview_artifact_type must be of type BenchmarkArenaArtifactType') + self._preview_artifact_type = preview_artifact_type + @property def update_mask(self) -> FieldMask: r""" @@ -2710,106 +3065,6 @@ def body_fields(): return '*' -class ApiValidationStage(KaggleObject): - r""" - The public API input shape of ValidationStage, which identifies the default - candidate by slug instead of id. - - Attributes: - type (BenchmarkTaskValidationStage) - Required. Not UNSPECIFIED. - default_model_version_slug (str) - Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default'). - On Harbor tasks, requires override_harness_version: without one the task - evaluates agents, so pick the model via default_agent_slug instead. - default_agent_slug (str) - Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high'). - default_model_version_config_slug (str) - Canonical BenchmarkModelVersionConfig.slug (e.g. - 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins - the reasoning effort. Harbor tasks only; requires - override_harness_version. - """ - - def __init__(self): - self._type = BenchmarkTaskValidationStage.BENCHMARK_TASK_VALIDATION_STAGE_UNSPECIFIED - self._default_model_version_slug = None - self._default_agent_slug = None - self._default_model_version_config_slug = None - self._freeze() - - @property - def type(self) -> 'BenchmarkTaskValidationStage': - """Required. Not UNSPECIFIED.""" - return self._type - - @type.setter - def type(self, type: 'BenchmarkTaskValidationStage'): - if type is None: - del self.type - return - if not isinstance(type, BenchmarkTaskValidationStage): - raise TypeError('type must be of type BenchmarkTaskValidationStage') - self._type = type - - @property - def default_model_version_slug(self) -> str: - r""" - Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default'). - On Harbor tasks, requires override_harness_version: without one the task - evaluates agents, so pick the model via default_agent_slug instead. - """ - return self._default_model_version_slug or "" - - @default_model_version_slug.setter - def default_model_version_slug(self, default_model_version_slug: str): - if default_model_version_slug is None: - del self.default_model_version_slug - return - if not isinstance(default_model_version_slug, str): - raise TypeError('default_model_version_slug must be of type str') - del self.default_agent_slug - del self.default_model_version_config_slug - self._default_model_version_slug = default_model_version_slug - - @property - def default_agent_slug(self) -> str: - """Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high').""" - return self._default_agent_slug or "" - - @default_agent_slug.setter - def default_agent_slug(self, default_agent_slug: str): - if default_agent_slug is None: - del self.default_agent_slug - return - if not isinstance(default_agent_slug, str): - raise TypeError('default_agent_slug must be of type str') - del self.default_model_version_slug - del self.default_model_version_config_slug - self._default_agent_slug = default_agent_slug - - @property - def default_model_version_config_slug(self) -> str: - r""" - Canonical BenchmarkModelVersionConfig.slug (e.g. - 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins - the reasoning effort. Harbor tasks only; requires - override_harness_version. - """ - return self._default_model_version_config_slug or "" - - @default_model_version_config_slug.setter - def default_model_version_config_slug(self, default_model_version_config_slug: str): - if default_model_version_config_slug is None: - del self.default_model_version_config_slug - return - if not isinstance(default_model_version_config_slug, str): - raise TypeError('default_model_version_config_slug must be of type str') - del self.default_model_version_slug - del self.default_agent_slug - self._default_model_version_config_slug = default_model_version_config_slug - - ApiBatchScheduleBenchmarkTaskRunsRequest._fields = [ FieldMetadata("taskSlugs", "task_slugs", "_task_slugs", ApiBenchmarkTaskSlug, [], ListSerializer(KaggleObjectSerializer())), FieldMetadata("modelVersionSlugs", "model_version_slugs", "_model_version_slugs", str, [], ListSerializer(PredefinedSerializer())), @@ -2853,6 +3108,9 @@ def default_model_version_config_slug(self, default_model_version_config_slug: s FieldMetadata("artifactPath", "artifact_path", "_artifact_path", str, None, PredefinedSerializer(), optional=True), FieldMetadata("artifactType", "artifact_type", "_artifact_type", BenchmarkArenaArtifactType, None, EnumSerializer(), optional=True), FieldMetadata("areRunArtifactsPublic", "are_run_artifacts_public", "_are_run_artifacts_public", bool, None, PredefinedSerializer(), optional=True), + FieldMetadata("previewArtifactPath", "preview_artifact_path", "_preview_artifact_path", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("previewArtifactType", "preview_artifact_type", "_preview_artifact_type", BenchmarkArenaArtifactType, None, EnumSerializer(), optional=True), + FieldMetadata("validationConfig", "validation_config", "_validation_config", ApiTaskCreationValidationConfig, None, KaggleObjectSerializer(), optional=True), ] ApiBenchmarkTaskCitation._fields = [ @@ -2860,6 +3118,20 @@ def default_model_version_config_slug(self, default_model_version_config_slug: s FieldMetadata("url", "url", "_url", str, "", PredefinedSerializer()), ] +ApiBenchmarkTaskCollaborator._fields = [ + FieldMetadata("principal", "principal", "_principal", ApiBenchmarkTaskPrincipal, None, KaggleObjectSerializer()), + FieldMetadata("role", "role", "_role", BenchmarkTaskCollaboratorRole, BenchmarkTaskCollaboratorRole.BENCHMARK_TASK_COLLABORATOR_ROLE_UNSPECIFIED, EnumSerializer()), +] + +ApiBenchmarkTaskCollaborators._fields = [ + FieldMetadata("collaborators", "collaborators", "_collaborators", ApiBenchmarkTaskCollaborator, [], ListSerializer(KaggleObjectSerializer())), +] + +ApiBenchmarkTaskPrincipal._fields = [ + FieldMetadata("username", "username", "_username", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("groupSlug", "group_slug", "_group_slug", str, None, PredefinedSerializer(), optional=True), +] + ApiBenchmarkTaskRun._fields = [ FieldMetadata("taskSlug", "task_slug", "_task_slug", ApiBenchmarkTaskSlug, None, KaggleObjectSerializer()), FieldMetadata("modelVersionSlug", "model_version_slug", "_model_version_slug", str, "", PredefinedSerializer()), @@ -2871,6 +3143,7 @@ def default_model_version_config_slug(self, default_model_version_config_slug: s FieldMetadata("agentSlug", "agent_slug", "_agent_slug", str, None, PredefinedSerializer(), optional=True), FieldMetadata("modelVersionConfigSlug", "model_version_config_slug", "_model_version_config_slug", str, None, PredefinedSerializer(), optional=True), FieldMetadata("result", "result", "_result", ApiBenchmarkTaskRunResult, None, KaggleObjectSerializer(), optional=True), + FieldMetadata("isManual", "is_manual", "_is_manual", bool, False, PredefinedSerializer()), ] ApiBenchmarkTaskRunIds._fields = [ @@ -2920,6 +3193,10 @@ def default_model_version_config_slug(self, default_model_version_config_slug: s ApiGetBenchmarkSchedulerStatusRequest._fields = [] +ApiGetBenchmarkTaskCollaboratorsRequest._fields = [ + FieldMetadata("slug", "slug", "_slug", ApiBenchmarkTaskSlug, None, KaggleObjectSerializer()), +] + ApiGetBenchmarkTaskQuotaRequest._fields = [] ApiGetBenchmarkTaskQuotaResponse._fields = [ @@ -2979,8 +3256,10 @@ def default_model_version_config_slug(self, default_model_version_config_slug: s FieldMetadata("publishRunArtifacts", "publish_run_artifacts", "_publish_run_artifacts", bool, False, PredefinedSerializer()), ] -ApiTaskCreationValidationConfig._fields = [ - FieldMetadata("validationStages", "validation_stages", "_validation_stages", ApiValidationStage, [], ListSerializer(KaggleObjectSerializer())), +ApiUpdateBenchmarkTaskCollaboratorsRequest._fields = [ + FieldMetadata("slug", "slug", "_slug", ApiBenchmarkTaskSlug, None, KaggleObjectSerializer()), + FieldMetadata("add", "add", "_add", ApiBenchmarkTaskCollaborator, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("remove", "remove", "_remove", ApiBenchmarkTaskPrincipal, [], ListSerializer(KaggleObjectSerializer())), ] ApiUpdateBenchmarkTaskRequest._fields = [ @@ -3005,12 +3284,7 @@ def default_model_version_config_slug(self, default_model_version_config_slug: s FieldMetadata("updateMask", "update_mask", "_update_mask", FieldMask, None, FieldMaskSerializer()), FieldMetadata("artifactPath", "artifact_path", "_artifact_path", str, None, PredefinedSerializer(), optional=True), FieldMetadata("artifactType", "artifact_type", "_artifact_type", BenchmarkArenaArtifactType, None, EnumSerializer(), optional=True), -] - -ApiValidationStage._fields = [ - FieldMetadata("type", "type", "_type", BenchmarkTaskValidationStage, BenchmarkTaskValidationStage.BENCHMARK_TASK_VALIDATION_STAGE_UNSPECIFIED, EnumSerializer()), - FieldMetadata("defaultModelVersionSlug", "default_model_version_slug", "_default_model_version_slug", str, None, PredefinedSerializer(), optional=True), - FieldMetadata("defaultAgentSlug", "default_agent_slug", "_default_agent_slug", str, None, PredefinedSerializer(), optional=True), - FieldMetadata("defaultModelVersionConfigSlug", "default_model_version_config_slug", "_default_model_version_config_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("previewArtifactPath", "preview_artifact_path", "_preview_artifact_path", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("previewArtifactType", "preview_artifact_type", "_preview_artifact_type", BenchmarkArenaArtifactType, None, EnumSerializer(), optional=True), ] diff --git a/kagglesdk/benchmarks/types/benchmark_types.py b/kagglesdk/benchmarks/types/benchmark_types.py index 16fb1c9..4e4b5bf 100644 --- a/kagglesdk/benchmarks/types/benchmark_types.py +++ b/kagglesdk/benchmarks/types/benchmark_types.py @@ -4,7 +4,7 @@ from kagglesdk.kaggle_object import * from kagglesdk.licenses.types.licenses_types import License from kagglesdk.users.types.legacy_organizations_service import OrganizationCard -from typing import Optional, List +from typing import List, Optional class BenchmarkTaskValidationStage(enum.Enum): r""" @@ -37,6 +37,139 @@ class HarborMetricType(enum.Enum): HARBOR_METRIC_TYPE_MAX = 3 HARBOR_METRIC_TYPE_MEAN = 4 +class ApiTaskCreationValidationConfig(KaggleObject): + r""" + The slug-keyed shape of TaskCreationValidationConfig, accepted by + CreateBenchmarkTask and echoed on BenchmarkTask and ApiBenchmarkTask reads. + Here rather than in benchmark_tasks_api_service.proto so the BenchmarkTask + read shape can use it. + + Attributes: + validation_stages (ApiValidationStage) + The validation stages to run against the task version. + """ + + def __init__(self): + self._validation_stages = [] + self._freeze() + + @property + def validation_stages(self) -> Optional[List[Optional['ApiValidationStage']]]: + """The validation stages to run against the task version.""" + return self._validation_stages + + @validation_stages.setter + def validation_stages(self, validation_stages: Optional[List[Optional['ApiValidationStage']]]): + if validation_stages is None: + del self.validation_stages + return + if not isinstance(validation_stages, list): + raise TypeError('validation_stages must be of type list') + if not all([isinstance(t, ApiValidationStage) for t in validation_stages]): + raise TypeError('validation_stages must contain only items of type ApiValidationStage') + self._validation_stages = validation_stages + + +class ApiValidationStage(KaggleObject): + r""" + The slug-keyed shape of ValidationStage, which identifies the default + candidate by slug instead of id. Accepted on create and echoed on reads. + + Attributes: + type (BenchmarkTaskValidationStage) + Required on create. Not UNSPECIFIED. + default_model_version_slug (str) + Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default'). + On Harbor tasks, requires override_harness_version: without one the task + evaluates agents, so pick the model via default_agent_slug instead. + default_agent_slug (str) + Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high'). + default_model_version_config_slug (str) + Canonical BenchmarkModelVersionConfig.slug (e.g. + 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins + the reasoning effort. Harbor tasks only; requires + override_harness_version. + """ + + def __init__(self): + self._type = BenchmarkTaskValidationStage.BENCHMARK_TASK_VALIDATION_STAGE_UNSPECIFIED + self._default_model_version_slug = None + self._default_agent_slug = None + self._default_model_version_config_slug = None + self._freeze() + + @property + def type(self) -> 'BenchmarkTaskValidationStage': + """Required on create. Not UNSPECIFIED.""" + return self._type + + @type.setter + def type(self, type: 'BenchmarkTaskValidationStage'): + if type is None: + del self.type + return + if not isinstance(type, BenchmarkTaskValidationStage): + raise TypeError('type must be of type BenchmarkTaskValidationStage') + self._type = type + + @property + def default_model_version_slug(self) -> str: + r""" + Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default'). + On Harbor tasks, requires override_harness_version: without one the task + evaluates agents, so pick the model via default_agent_slug instead. + """ + return self._default_model_version_slug or "" + + @default_model_version_slug.setter + def default_model_version_slug(self, default_model_version_slug: str): + if default_model_version_slug is None: + del self.default_model_version_slug + return + if not isinstance(default_model_version_slug, str): + raise TypeError('default_model_version_slug must be of type str') + del self.default_agent_slug + del self.default_model_version_config_slug + self._default_model_version_slug = default_model_version_slug + + @property + def default_agent_slug(self) -> str: + """Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high').""" + return self._default_agent_slug or "" + + @default_agent_slug.setter + def default_agent_slug(self, default_agent_slug: str): + if default_agent_slug is None: + del self.default_agent_slug + return + if not isinstance(default_agent_slug, str): + raise TypeError('default_agent_slug must be of type str') + del self.default_model_version_slug + del self.default_model_version_config_slug + self._default_agent_slug = default_agent_slug + + @property + def default_model_version_config_slug(self) -> str: + r""" + Canonical BenchmarkModelVersionConfig.slug (e.g. + 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins + the reasoning effort. Harbor tasks only; requires + override_harness_version. + """ + return self._default_model_version_config_slug or "" + + @default_model_version_config_slug.setter + def default_model_version_config_slug(self, default_model_version_config_slug: str): + if default_model_version_config_slug is None: + del self.default_model_version_config_slug + return + if not isinstance(default_model_version_config_slug, str): + raise TypeError('default_model_version_config_slug must be of type str') + del self.default_model_version_slug + del self.default_agent_slug + self._default_model_version_config_slug = default_model_version_config_slug + + class BenchmarkArenaCombination(KaggleObject): r""" A (task version, candidate) combination opted into a benchmark version's @@ -83,18 +216,25 @@ def candidate(self, candidate: Optional['BenchmarkCandidate']): class BenchmarkArenaSettings(KaggleObject): r""" - Human-vote arena over a benchmark version's task run artifacts. + Human-vote arena over a benchmark version's task run artifacts. Everyone who + can read the benchmark version sees these, on + BenchmarkVersion.arena_settings, so only fields fit for its readers belong + here. Attributes: artifact_path (str) - Glob, relative to a run's session output root, selecting the artifact, + Unused: arenas show the artifact each task version's own artifact_path + selects. Still validated as a glob relative to a run's session output root, e.g. '*.mp4'. artifact_type (BenchmarkArenaArtifactType) + Labels and filters the arena. Each task version's own artifact_type decides + how its outputs are shown. reveal_candidates (bool) Show candidate names after a vote. Off keeps voting fully blind. voting_enabled (bool) Kill switches for the arena's public surfaces. Off when the arena is - created. + created. The gallery labels each output with its candidate, so with + voting on it requires reveal_candidates. gallery_enabled (bool) arena_task_version_id (int) The task version the arena's overall Elo is written to, shown as a @@ -104,6 +244,15 @@ class BenchmarkArenaSettings(KaggleObject): Clicking an artifact opens it full size, rather than voting for it in the arena. HTML artifacts always do; this extends it to every type, for outputs too detailed to judge at card size. + show_vote_counts (bool) + Show each candidate's vote count, and the total, on the arena's + leaderboard. Only votes behind the Elos count. + votes_task_version_id (int) + The task version each candidate's vote count is written to, a leaderboard + column beside the Elo. Created with the Elo's. + vote_count (int) + How many votes the arena's Elos were last computed from. Set only when + show_vote_counts is on. """ def __init__(self): @@ -115,12 +264,16 @@ def __init__(self): self._arena_task_version_id = None self._gallery_card_size = BenchmarkGalleryCardSize.BENCHMARK_GALLERY_CARD_SIZE_UNSPECIFIED self._expand_artifacts = False + self._show_vote_counts = False + self._votes_task_version_id = None + self._vote_count = None self._freeze() @property def artifact_path(self) -> str: r""" - Glob, relative to a run's session output root, selecting the artifact, + Unused: arenas show the artifact each task version's own artifact_path + selects. Still validated as a glob relative to a run's session output root, e.g. '*.mp4'. """ return self._artifact_path @@ -136,6 +289,10 @@ def artifact_path(self, artifact_path: str): @property def artifact_type(self) -> 'BenchmarkArenaArtifactType': + r""" + Labels and filters the arena. Each task version's own artifact_type decides + how its outputs are shown. + """ return self._artifact_type @artifact_type.setter @@ -165,7 +322,8 @@ def reveal_candidates(self, reveal_candidates: bool): def voting_enabled(self) -> bool: r""" Kill switches for the arena's public surfaces. Off when the arena is - created. + created. The gallery labels each output with its candidate, so with + voting on it requires reveal_candidates. """ return self._voting_enabled @@ -239,6 +397,57 @@ def expand_artifacts(self, expand_artifacts: bool): raise TypeError('expand_artifacts must be of type bool') self._expand_artifacts = expand_artifacts + @property + def show_vote_counts(self) -> bool: + r""" + Show each candidate's vote count, and the total, on the arena's + leaderboard. Only votes behind the Elos count. + """ + return self._show_vote_counts + + @show_vote_counts.setter + def show_vote_counts(self, show_vote_counts: bool): + if show_vote_counts is None: + del self.show_vote_counts + return + if not isinstance(show_vote_counts, bool): + raise TypeError('show_vote_counts must be of type bool') + self._show_vote_counts = show_vote_counts + + @property + def votes_task_version_id(self) -> int: + r""" + The task version each candidate's vote count is written to, a leaderboard + column beside the Elo. Created with the Elo's. + """ + return self._votes_task_version_id or 0 + + @votes_task_version_id.setter + def votes_task_version_id(self, votes_task_version_id: Optional[int]): + if votes_task_version_id is None: + del self.votes_task_version_id + return + if not isinstance(votes_task_version_id, int): + raise TypeError('votes_task_version_id must be of type int') + self._votes_task_version_id = votes_task_version_id + + @property + def vote_count(self) -> int: + r""" + How many votes the arena's Elos were last computed from. Set only when + show_vote_counts is on. + """ + return self._vote_count or 0 + + @vote_count.setter + def vote_count(self, vote_count: Optional[int]): + if vote_count is None: + del self.vote_count + return + if not isinstance(vote_count, int): + raise TypeError('vote_count must be of type int') + self._vote_count = vote_count + class BenchmarkCandidate(KaggleObject): r""" @@ -335,6 +544,197 @@ def reasoning_effort(self, reasoning_effort: Optional[str]): self._reasoning_effort = reasoning_effort +class BenchmarkMetricProducer(KaggleObject): + r""" + Attributes: + id (int) + benchmark_version_id (int) + definition_dataset_version_id (int) + definition_dataset_version_subpath (str) + entrypoint (str) + input_options (BenchmarkMetricProducerInputOptions) + creator_user_id (int) + create_time (datetime) + """ + + def __init__(self): + self._id = 0 + self._benchmark_version_id = 0 + self._definition_dataset_version_id = 0 + self._definition_dataset_version_subpath = None + self._entrypoint = "" + self._input_options = None + self._creator_user_id = 0 + self._create_time = None + self._freeze() + + @property + def id(self) -> int: + return self._id + + @id.setter + def id(self, id: int): + if id is None: + del self.id + return + if not isinstance(id, int): + raise TypeError('id must be of type int') + self._id = id + + @property + def benchmark_version_id(self) -> int: + return self._benchmark_version_id + + @benchmark_version_id.setter + def benchmark_version_id(self, benchmark_version_id: int): + if benchmark_version_id is None: + del self.benchmark_version_id + return + if not isinstance(benchmark_version_id, int): + raise TypeError('benchmark_version_id must be of type int') + self._benchmark_version_id = benchmark_version_id + + @property + def definition_dataset_version_id(self) -> int: + return self._definition_dataset_version_id + + @definition_dataset_version_id.setter + def definition_dataset_version_id(self, definition_dataset_version_id: int): + if definition_dataset_version_id is None: + del self.definition_dataset_version_id + return + if not isinstance(definition_dataset_version_id, int): + raise TypeError('definition_dataset_version_id must be of type int') + self._definition_dataset_version_id = definition_dataset_version_id + + @property + def definition_dataset_version_subpath(self) -> str: + return self._definition_dataset_version_subpath or "" + + @definition_dataset_version_subpath.setter + def definition_dataset_version_subpath(self, definition_dataset_version_subpath: Optional[str]): + if definition_dataset_version_subpath is None: + del self.definition_dataset_version_subpath + return + if not isinstance(definition_dataset_version_subpath, str): + raise TypeError('definition_dataset_version_subpath must be of type str') + self._definition_dataset_version_subpath = definition_dataset_version_subpath + + @property + def entrypoint(self) -> str: + return self._entrypoint + + @entrypoint.setter + def entrypoint(self, entrypoint: str): + if entrypoint is None: + del self.entrypoint + return + if not isinstance(entrypoint, str): + raise TypeError('entrypoint must be of type str') + self._entrypoint = entrypoint + + @property + def input_options(self) -> Optional['BenchmarkMetricProducerInputOptions']: + return self._input_options + + @input_options.setter + def input_options(self, input_options: Optional['BenchmarkMetricProducerInputOptions']): + if input_options is None: + del self.input_options + return + if not isinstance(input_options, BenchmarkMetricProducerInputOptions): + raise TypeError('input_options must be of type BenchmarkMetricProducerInputOptions') + self._input_options = input_options + + @property + def creator_user_id(self) -> int: + return self._creator_user_id + + @creator_user_id.setter + def creator_user_id(self, creator_user_id: int): + if creator_user_id is None: + del self.creator_user_id + return + if not isinstance(creator_user_id, int): + raise TypeError('creator_user_id must be of type int') + self._creator_user_id = creator_user_id + + @property + def create_time(self) -> datetime: + return self._create_time + + @create_time.setter + def create_time(self, create_time: datetime): + if create_time is None: + del self.create_time + return + if not isinstance(create_time, datetime): + raise TypeError('create_time must be of type datetime') + self._create_time = create_time + + +class BenchmarkMetricProducerInputOptions(KaggleObject): + r""" + Shapes what a BenchmarkMetricProducer is given from the benchmark version's + runs. Persisted as JSON on BenchmarkMetricProducers. + + Attributes: + results_only (bool) + Stage only the manifest of scores and cost/token usage, which come from + SQL, and skip copying run files. Per-trial rewards exist only in each + trial's result.json, so they are omitted; producers that need them should + copy result.json via output_path_regex instead. + output_path_regex (str) + Copy only run files whose path, relative to each run's session output + directory, matches this regex. Empty copies all files. Ignored when + results_only is set. Agent credential files under agent/sessions/ are + never copied, whatever the regex. + """ + + def __init__(self): + self._results_only = False + self._output_path_regex = "" + self._freeze() + + @property + def results_only(self) -> bool: + r""" + Stage only the manifest of scores and cost/token usage, which come from + SQL, and skip copying run files. Per-trial rewards exist only in each + trial's result.json, so they are omitted; producers that need them should + copy result.json via output_path_regex instead. + """ + return self._results_only + + @results_only.setter + def results_only(self, results_only: bool): + if results_only is None: + del self.results_only + return + if not isinstance(results_only, bool): + raise TypeError('results_only must be of type bool') + self._results_only = results_only + + @property + def output_path_regex(self) -> str: + r""" + Copy only run files whose path, relative to each run's session output + directory, matches this regex. Empty copies all files. Ignored when + results_only is set. Agent credential files under agent/sessions/ are + never copied, whatever the regex. + """ + return self._output_path_regex + + @output_path_regex.setter + def output_path_regex(self, output_path_regex: str): + if output_path_regex is None: + del self.output_path_regex + return + if not isinstance(output_path_regex, str): + raise TypeError('output_path_regex must be of type str') + self._output_path_regex = output_path_regex + + class BenchmarkModel(KaggleObject): r""" Attributes: @@ -1131,6 +1531,24 @@ class BenchmarkSchedulerSettings(KaggleObject): global_min_model_proxy_priority (float) Minimum priority value sent to Model Proxy for every run. 0.0 is most urgent, so this caps how urgent a run can be. Unset = no floor. + unattributed_scheduler_priority (float) + Scheduler tier for queued runs attributed to no benchmark, on the same + [0.0, 4.0] scale as Benchmark.scheduler_priority. Read live by the drain, + so a change applies to runs already queued. Unset = Normal (2.0). + user_session_caps_enforced (bool) + When on, the drain leaves runs Queued that would take their creator past + their KKB session cap (KkbQuotaTiers / KkbUserQuotas), and BatchSchedule + rejects such requests with ResourceExhausted. When off, both start the runs + and only report it. + Optional so a client that omits it doesn't turn enforcement off. + kkb_pool_gating_enabled (bool) + Whether the drain holds back runs whose KKB pool has no room left, per + max_kkb_pool_queued_sessions. Off, the drain still measures the pools and + reports how many runs it would have held back. + max_kkb_pool_queued_sessions (int) + How many sessions a KKB pool may have waiting for a VM before the drain + stops starting runs on it. Unset = 100; 0 fills only idle VMs. Pools scale + up on their queue, so a low cap slows how fast they grow. """ def __init__(self): @@ -1148,6 +1566,10 @@ def __init__(self): self._max_retries_per_tick = None self._screening_enabled = None self._global_min_model_proxy_priority = None + self._unattributed_scheduler_priority = None + self._user_session_caps_enforced = None + self._kkb_pool_gating_enabled = False + self._max_kkb_pool_queued_sessions = None self._freeze() @property @@ -1384,6 +1806,80 @@ def global_min_model_proxy_priority(self, global_min_model_proxy_priority: Optio raise TypeError('global_min_model_proxy_priority must be of type float') self._global_min_model_proxy_priority = global_min_model_proxy_priority + @property + def unattributed_scheduler_priority(self) -> float: + r""" + Scheduler tier for queued runs attributed to no benchmark, on the same + [0.0, 4.0] scale as Benchmark.scheduler_priority. Read live by the drain, + so a change applies to runs already queued. Unset = Normal (2.0). + """ + return self._unattributed_scheduler_priority or 0.0 + + @unattributed_scheduler_priority.setter + def unattributed_scheduler_priority(self, unattributed_scheduler_priority: Optional[float]): + if unattributed_scheduler_priority is None: + del self.unattributed_scheduler_priority + return + if not isinstance(unattributed_scheduler_priority, float): + raise TypeError('unattributed_scheduler_priority must be of type float') + self._unattributed_scheduler_priority = unattributed_scheduler_priority + + @property + def user_session_caps_enforced(self) -> bool: + r""" + When on, the drain leaves runs Queued that would take their creator past + their KKB session cap (KkbQuotaTiers / KkbUserQuotas), and BatchSchedule + rejects such requests with ResourceExhausted. When off, both start the runs + and only report it. + Optional so a client that omits it doesn't turn enforcement off. + """ + return self._user_session_caps_enforced or False + + @user_session_caps_enforced.setter + def user_session_caps_enforced(self, user_session_caps_enforced: Optional[bool]): + if user_session_caps_enforced is None: + del self.user_session_caps_enforced + return + if not isinstance(user_session_caps_enforced, bool): + raise TypeError('user_session_caps_enforced must be of type bool') + self._user_session_caps_enforced = user_session_caps_enforced + + @property + def kkb_pool_gating_enabled(self) -> bool: + r""" + Whether the drain holds back runs whose KKB pool has no room left, per + max_kkb_pool_queued_sessions. Off, the drain still measures the pools and + reports how many runs it would have held back. + """ + return self._kkb_pool_gating_enabled + + @kkb_pool_gating_enabled.setter + def kkb_pool_gating_enabled(self, kkb_pool_gating_enabled: bool): + if kkb_pool_gating_enabled is None: + del self.kkb_pool_gating_enabled + return + if not isinstance(kkb_pool_gating_enabled, bool): + raise TypeError('kkb_pool_gating_enabled must be of type bool') + self._kkb_pool_gating_enabled = kkb_pool_gating_enabled + + @property + def max_kkb_pool_queued_sessions(self) -> int: + r""" + How many sessions a KKB pool may have waiting for a VM before the drain + stops starting runs on it. Unset = 100; 0 fills only idle VMs. Pools scale + up on their queue, so a low cap slows how fast they grow. + """ + return self._max_kkb_pool_queued_sessions or 0 + + @max_kkb_pool_queued_sessions.setter + def max_kkb_pool_queued_sessions(self, max_kkb_pool_queued_sessions: Optional[int]): + if max_kkb_pool_queued_sessions is None: + del self.max_kkb_pool_queued_sessions + return + if not isinstance(max_kkb_pool_queued_sessions, int): + raise TypeError('max_kkb_pool_queued_sessions must be of type int') + self._max_kkb_pool_queued_sessions = max_kkb_pool_queued_sessions + class BenchmarkTaskDefinition(KaggleObject): r""" @@ -1726,6 +2222,9 @@ class BenchmarkTaskKaggleDatasetsDefinition(KaggleObject): Canonical BenchmarkModelVersionConfig.slug (e.g. 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins the reasoning effort. Requires override_harness_version. + environment (HarborTaskEnvironment) + What the task's task.toml declares about where it runs. Ignored on create, + which reads these from task.toml itself. """ def __init__(self): @@ -1743,6 +2242,7 @@ def __init__(self): self._metrics = [] self._session_concurrency = None self._default_model_version_config_slug = None + self._environment = None self._freeze() @property @@ -1998,6 +2498,23 @@ def session_concurrency(self, session_concurrency: Optional[int]): raise TypeError('session_concurrency must be of type int') self._session_concurrency = session_concurrency + @property + def environment(self) -> Optional['HarborTaskEnvironment']: + r""" + What the task's task.toml declares about where it runs. Ignored on create, + which reads these from task.toml itself. + """ + return self._environment + + @environment.setter + def environment(self, environment: Optional['HarborTaskEnvironment']): + if environment is None: + del self.environment + return + if not isinstance(environment, HarborTaskEnvironment): + raise TypeError('environment must be of type HarborTaskEnvironment') + self._environment = environment + class BenchmarkTaskNotebookDefinition(KaggleObject): r""" @@ -2101,11 +2618,18 @@ class DatasetMount(KaggleObject): Absolute path inside the container where the dataset is mounted. Callers pick this so tasks can locate their data anywhere. Unique across the mount list; must not shadow the task-definition mount. + dataset_id (int) + As on DefinitionSource. + dataset_version_id (int) + databundle_version_id (int) """ def __init__(self): self._dataset_version_slug = "" self._mount_path = "" + self._dataset_id = 0 + self._dataset_version_id = 0 + self._databundle_version_id = 0 self._freeze() @property @@ -2143,6 +2667,46 @@ def mount_path(self, mount_path: str): raise TypeError('mount_path must be of type str') self._mount_path = mount_path + @property + def dataset_id(self) -> int: + """As on DefinitionSource.""" + return self._dataset_id + + @dataset_id.setter + def dataset_id(self, dataset_id: int): + if dataset_id is None: + del self.dataset_id + return + if not isinstance(dataset_id, int): + raise TypeError('dataset_id must be of type int') + self._dataset_id = dataset_id + + @property + def dataset_version_id(self) -> int: + return self._dataset_version_id + + @dataset_version_id.setter + def dataset_version_id(self, dataset_version_id: int): + if dataset_version_id is None: + del self.dataset_version_id + return + if not isinstance(dataset_version_id, int): + raise TypeError('dataset_version_id must be of type int') + self._dataset_version_id = dataset_version_id + + @property + def databundle_version_id(self) -> int: + return self._databundle_version_id + + @databundle_version_id.setter + def databundle_version_id(self, databundle_version_id: int): + if databundle_version_id is None: + del self.databundle_version_id + return + if not isinstance(databundle_version_id, int): + raise TypeError('databundle_version_id must be of type int') + self._databundle_version_id = databundle_version_id + class DefinitionSource(KaggleObject): r""" @@ -2156,11 +2720,19 @@ class DefinitionSource(KaggleObject): sub_path (str) Sub-directory within the dataset containing task.toml. Empty means the dataset root. + dataset_id (int) + The pinned version's ids, for embedding its file explorer. Ignored on + create, which takes dataset_version_slug. + dataset_version_id (int) + databundle_version_id (int) """ def __init__(self): self._dataset_version_slug = "" self._sub_path = "" + self._dataset_id = 0 + self._dataset_version_id = 0 + self._databundle_version_id = 0 self._freeze() @property @@ -2197,6 +2769,49 @@ def sub_path(self, sub_path: str): raise TypeError('sub_path must be of type str') self._sub_path = sub_path + @property + def dataset_id(self) -> int: + r""" + The pinned version's ids, for embedding its file explorer. Ignored on + create, which takes dataset_version_slug. + """ + return self._dataset_id + + @dataset_id.setter + def dataset_id(self, dataset_id: int): + if dataset_id is None: + del self.dataset_id + return + if not isinstance(dataset_id, int): + raise TypeError('dataset_id must be of type int') + self._dataset_id = dataset_id + + @property + def dataset_version_id(self) -> int: + return self._dataset_version_id + + @dataset_version_id.setter + def dataset_version_id(self, dataset_version_id: int): + if dataset_version_id is None: + del self.dataset_version_id + return + if not isinstance(dataset_version_id, int): + raise TypeError('dataset_version_id must be of type int') + self._dataset_version_id = dataset_version_id + + @property + def databundle_version_id(self) -> int: + return self._databundle_version_id + + @databundle_version_id.setter + def databundle_version_id(self, databundle_version_id: int): + if databundle_version_id is None: + del self.databundle_version_id + return + if not isinstance(databundle_version_id, int): + raise TypeError('databundle_version_id must be of type int') + self._databundle_version_id = databundle_version_id + class EnvVariable(KaggleObject): r""" @@ -2261,6 +2876,161 @@ def is_secret_name(self, is_secret_name: bool): self._is_secret_name = is_secret_name +class HarborTaskEnvironment(KaggleObject): + r""" + The resources and network access a Harbor task's task.toml declares, as read + when its version was created. Unset fields weren't declared. + + Attributes: + cpus (int) + memory_mb (int) + storage_mb (int) + gpus (int) + gpu_types (str) + tpu_type (str) + tpu_topology (str) + agent_network_mode (str) + Harbor's network_mode for the agent container, collapsed to its most + permissive step. + verifier_network_mode (str) + """ + + def __init__(self): + self._cpus = None + self._memory_mb = None + self._storage_mb = None + self._gpus = None + self._gpu_types = [] + self._tpu_type = None + self._tpu_topology = None + self._agent_network_mode = None + self._verifier_network_mode = None + self._freeze() + + @property + def cpus(self) -> int: + return self._cpus or 0 + + @cpus.setter + def cpus(self, cpus: Optional[int]): + if cpus is None: + del self.cpus + return + if not isinstance(cpus, int): + raise TypeError('cpus must be of type int') + self._cpus = cpus + + @property + def memory_mb(self) -> int: + return self._memory_mb or 0 + + @memory_mb.setter + def memory_mb(self, memory_mb: Optional[int]): + if memory_mb is None: + del self.memory_mb + return + if not isinstance(memory_mb, int): + raise TypeError('memory_mb must be of type int') + self._memory_mb = memory_mb + + @property + def storage_mb(self) -> int: + return self._storage_mb or 0 + + @storage_mb.setter + def storage_mb(self, storage_mb: Optional[int]): + if storage_mb is None: + del self.storage_mb + return + if not isinstance(storage_mb, int): + raise TypeError('storage_mb must be of type int') + self._storage_mb = storage_mb + + @property + def gpus(self) -> int: + return self._gpus or 0 + + @gpus.setter + def gpus(self, gpus: Optional[int]): + if gpus is None: + del self.gpus + return + if not isinstance(gpus, int): + raise TypeError('gpus must be of type int') + self._gpus = gpus + + @property + def gpu_types(self) -> Optional[List[str]]: + return self._gpu_types + + @gpu_types.setter + def gpu_types(self, gpu_types: Optional[List[str]]): + if gpu_types is None: + del self.gpu_types + return + if not isinstance(gpu_types, list): + raise TypeError('gpu_types must be of type list') + if not all([isinstance(t, str) for t in gpu_types]): + raise TypeError('gpu_types must contain only items of type str') + self._gpu_types = gpu_types + + @property + def tpu_type(self) -> str: + return self._tpu_type or "" + + @tpu_type.setter + def tpu_type(self, tpu_type: Optional[str]): + if tpu_type is None: + del self.tpu_type + return + if not isinstance(tpu_type, str): + raise TypeError('tpu_type must be of type str') + self._tpu_type = tpu_type + + @property + def tpu_topology(self) -> str: + return self._tpu_topology or "" + + @tpu_topology.setter + def tpu_topology(self, tpu_topology: Optional[str]): + if tpu_topology is None: + del self.tpu_topology + return + if not isinstance(tpu_topology, str): + raise TypeError('tpu_topology must be of type str') + self._tpu_topology = tpu_topology + + @property + def agent_network_mode(self) -> str: + r""" + Harbor's network_mode for the agent container, collapsed to its most + permissive step. + """ + return self._agent_network_mode or "" + + @agent_network_mode.setter + def agent_network_mode(self, agent_network_mode: Optional[str]): + if agent_network_mode is None: + del self.agent_network_mode + return + if not isinstance(agent_network_mode, str): + raise TypeError('agent_network_mode must be of type str') + self._agent_network_mode = agent_network_mode + + @property + def verifier_network_mode(self) -> str: + return self._verifier_network_mode or "" + + @verifier_network_mode.setter + def verifier_network_mode(self, verifier_network_mode: Optional[str]): + if verifier_network_mode is None: + del self.verifier_network_mode + return + if not isinstance(verifier_network_mode, str): + raise TypeError('verifier_network_mode must be of type str') + self._verifier_network_mode = verifier_network_mode + + class NumericResult(KaggleObject): r""" Attributes: @@ -2463,6 +3233,17 @@ def override_existing(self, override_existing: bool): self._override_existing = override_existing +ApiTaskCreationValidationConfig._fields = [ + FieldMetadata("validationStages", "validation_stages", "_validation_stages", ApiValidationStage, [], ListSerializer(KaggleObjectSerializer())), +] + +ApiValidationStage._fields = [ + FieldMetadata("type", "type", "_type", BenchmarkTaskValidationStage, BenchmarkTaskValidationStage.BENCHMARK_TASK_VALIDATION_STAGE_UNSPECIFIED, EnumSerializer()), + FieldMetadata("defaultModelVersionSlug", "default_model_version_slug", "_default_model_version_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("defaultAgentSlug", "default_agent_slug", "_default_agent_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("defaultModelVersionConfigSlug", "default_model_version_config_slug", "_default_model_version_config_slug", str, None, PredefinedSerializer(), optional=True), +] + BenchmarkArenaCombination._fields = [ FieldMetadata("taskVersionId", "task_version_id", "_task_version_id", int, 0, PredefinedSerializer()), FieldMetadata("candidate", "candidate", "_candidate", BenchmarkCandidate, None, KaggleObjectSerializer()), @@ -2477,6 +3258,9 @@ def override_existing(self, override_existing: bool): FieldMetadata("arenaTaskVersionId", "arena_task_version_id", "_arena_task_version_id", int, None, PredefinedSerializer(), optional=True), FieldMetadata("galleryCardSize", "gallery_card_size", "_gallery_card_size", BenchmarkGalleryCardSize, BenchmarkGalleryCardSize.BENCHMARK_GALLERY_CARD_SIZE_UNSPECIFIED, EnumSerializer()), FieldMetadata("expandArtifacts", "expand_artifacts", "_expand_artifacts", bool, False, PredefinedSerializer()), + FieldMetadata("showVoteCounts", "show_vote_counts", "_show_vote_counts", bool, False, PredefinedSerializer()), + FieldMetadata("votesTaskVersionId", "votes_task_version_id", "_votes_task_version_id", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("voteCount", "vote_count", "_vote_count", int, None, PredefinedSerializer(), optional=True), ] BenchmarkCandidate._fields = [ @@ -2486,6 +3270,22 @@ def override_existing(self, override_existing: bool): FieldMetadata("benchmarkModelVersionConfigId", "benchmark_model_version_config_id", "_benchmark_model_version_config_id", int, None, PredefinedSerializer(), optional=True), ] +BenchmarkMetricProducer._fields = [ + FieldMetadata("id", "id", "_id", int, 0, PredefinedSerializer()), + FieldMetadata("benchmarkVersionId", "benchmark_version_id", "_benchmark_version_id", int, 0, PredefinedSerializer()), + FieldMetadata("definitionDatasetVersionId", "definition_dataset_version_id", "_definition_dataset_version_id", int, 0, PredefinedSerializer()), + FieldMetadata("definitionDatasetVersionSubpath", "definition_dataset_version_subpath", "_definition_dataset_version_subpath", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("entrypoint", "entrypoint", "_entrypoint", str, "", PredefinedSerializer()), + FieldMetadata("inputOptions", "input_options", "_input_options", BenchmarkMetricProducerInputOptions, None, KaggleObjectSerializer()), + FieldMetadata("creatorUserId", "creator_user_id", "_creator_user_id", int, 0, PredefinedSerializer()), + FieldMetadata("createTime", "create_time", "_create_time", datetime, None, DateTimeSerializer()), +] + +BenchmarkMetricProducerInputOptions._fields = [ + FieldMetadata("resultsOnly", "results_only", "_results_only", bool, False, PredefinedSerializer()), + FieldMetadata("outputPathRegex", "output_path_regex", "_output_path_regex", str, "", PredefinedSerializer()), +] + BenchmarkModel._fields = [ FieldMetadata("id", "id", "_id", int, 0, PredefinedSerializer()), FieldMetadata("displayName", "display_name", "_display_name", str, "", PredefinedSerializer()), @@ -2549,6 +3349,10 @@ def override_existing(self, override_existing: bool): FieldMetadata("maxRetriesPerTick", "max_retries_per_tick", "_max_retries_per_tick", int, None, PredefinedSerializer(), optional=True), FieldMetadata("screeningEnabled", "screening_enabled", "_screening_enabled", bool, None, PredefinedSerializer(), optional=True), FieldMetadata("globalMinModelProxyPriority", "global_min_model_proxy_priority", "_global_min_model_proxy_priority", float, None, PredefinedSerializer(), optional=True), + FieldMetadata("unattributedSchedulerPriority", "unattributed_scheduler_priority", "_unattributed_scheduler_priority", float, None, PredefinedSerializer(), optional=True), + FieldMetadata("userSessionCapsEnforced", "user_session_caps_enforced", "_user_session_caps_enforced", bool, None, PredefinedSerializer(), optional=True), + FieldMetadata("kkbPoolGatingEnabled", "kkb_pool_gating_enabled", "_kkb_pool_gating_enabled", bool, False, PredefinedSerializer()), + FieldMetadata("maxKkbPoolQueuedSessions", "max_kkb_pool_queued_sessions", "_max_kkb_pool_queued_sessions", int, None, PredefinedSerializer(), optional=True), ] BenchmarkTaskDefinition._fields = [ @@ -2585,6 +3389,7 @@ def override_existing(self, override_existing: bool): FieldMetadata("metrics", "metrics", "_metrics", HarborMetricType, [], ListSerializer(EnumSerializer())), FieldMetadata("sessionConcurrency", "session_concurrency", "_session_concurrency", int, None, PredefinedSerializer(), optional=True), FieldMetadata("defaultModelVersionConfigSlug", "default_model_version_config_slug", "_default_model_version_config_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("environment", "environment", "_environment", HarborTaskEnvironment, None, KaggleObjectSerializer()), ] BenchmarkTaskNotebookDefinition._fields = [] @@ -2601,11 +3406,17 @@ def override_existing(self, override_existing: bool): DatasetMount._fields = [ FieldMetadata("datasetVersionSlug", "dataset_version_slug", "_dataset_version_slug", str, "", PredefinedSerializer()), FieldMetadata("mountPath", "mount_path", "_mount_path", str, "", PredefinedSerializer()), + FieldMetadata("datasetId", "dataset_id", "_dataset_id", int, 0, PredefinedSerializer()), + FieldMetadata("datasetVersionId", "dataset_version_id", "_dataset_version_id", int, 0, PredefinedSerializer()), + FieldMetadata("databundleVersionId", "databundle_version_id", "_databundle_version_id", int, 0, PredefinedSerializer()), ] DefinitionSource._fields = [ FieldMetadata("datasetVersionSlug", "dataset_version_slug", "_dataset_version_slug", str, "", PredefinedSerializer()), FieldMetadata("subPath", "sub_path", "_sub_path", str, "", PredefinedSerializer()), + FieldMetadata("datasetId", "dataset_id", "_dataset_id", int, 0, PredefinedSerializer()), + FieldMetadata("datasetVersionId", "dataset_version_id", "_dataset_version_id", int, 0, PredefinedSerializer()), + FieldMetadata("databundleVersionId", "databundle_version_id", "_databundle_version_id", int, 0, PredefinedSerializer()), ] EnvVariable._fields = [ @@ -2614,6 +3425,18 @@ def override_existing(self, override_existing: bool): FieldMetadata("isSecretName", "is_secret_name", "_is_secret_name", bool, False, PredefinedSerializer()), ] +HarborTaskEnvironment._fields = [ + FieldMetadata("cpus", "cpus", "_cpus", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("memoryMb", "memory_mb", "_memory_mb", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("storageMb", "storage_mb", "_storage_mb", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("gpus", "gpus", "_gpus", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("gpuTypes", "gpu_types", "_gpu_types", str, [], ListSerializer(PredefinedSerializer())), + FieldMetadata("tpuType", "tpu_type", "_tpu_type", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("tpuTopology", "tpu_topology", "_tpu_topology", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("agentNetworkMode", "agent_network_mode", "_agent_network_mode", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("verifierNetworkMode", "verifier_network_mode", "_verifier_network_mode", str, None, PredefinedSerializer(), optional=True), +] + NumericResult._fields = [ FieldMetadata("value", "value", "_value", float, 0.0, PredefinedSerializer()), FieldMetadata("confidenceInterval", "confidence_interval", "_confidence_interval", float, None, PredefinedSerializer(), optional=True), diff --git a/kagglesdk/benchmarks/types/benchmarks_api_service.py b/kagglesdk/benchmarks/types/benchmarks_api_service.py index aceb649..24fed6b 100644 --- a/kagglesdk/benchmarks/types/benchmarks_api_service.py +++ b/kagglesdk/benchmarks/types/benchmarks_api_service.py @@ -1,7 +1,7 @@ from google.protobuf.field_mask_pb2 import FieldMask from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkCandidateClass, BenchmarkLeaderboardDisplayType, BenchmarkVersionAgentMappingType from kagglesdk.benchmarks.types.benchmark_tasks_api_service import ApiBenchmarkTaskSlug -from kagglesdk.benchmarks.types.benchmark_types import BenchmarkArenaCombination, BenchmarkArenaSettings, BenchmarkModel, BenchmarkResult +from kagglesdk.benchmarks.types.benchmark_types import BenchmarkArenaCombination, BenchmarkArenaSettings, BenchmarkMetricProducerInputOptions, BenchmarkModel, BenchmarkResult, DatasetMount, DefinitionSource, EnvVariable, UserSecret from kagglesdk.kaggle_object import * from typing import Optional, List @@ -154,6 +154,127 @@ def skippedTasks(self): return self.skipped_tasks +class ApiBatchScheduleBenchmarkMetricsRequest(KaggleObject): + r""" + Attributes: + owner_slug (str) + benchmark_slug (str) + version_number (int) + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + benchmark_metric_producer_id (int) + Any of the version's producers, not only its latest. + """ + + def __init__(self): + self._owner_slug = "" + self._benchmark_slug = "" + self._version_number = None + self._benchmark_metric_producer_id = 0 + self._freeze() + + @property + def owner_slug(self) -> str: + return self._owner_slug + + @owner_slug.setter + def owner_slug(self, owner_slug: str): + if owner_slug is None: + del self.owner_slug + return + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug + + @property + def benchmark_slug(self) -> str: + return self._benchmark_slug + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def benchmark_metric_producer_id(self) -> int: + """Any of the version's producers, not only its latest.""" + return self._benchmark_metric_producer_id + + @benchmark_metric_producer_id.setter + def benchmark_metric_producer_id(self, benchmark_metric_producer_id: int): + if benchmark_metric_producer_id is None: + del self.benchmark_metric_producer_id + return + if not isinstance(benchmark_metric_producer_id, int): + raise TypeError('benchmark_metric_producer_id must be of type int') + self._benchmark_metric_producer_id = benchmark_metric_producer_id + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/versions/{version_number}/metrics/schedule' + else: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/metrics/schedule' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + +class ApiBatchScheduleBenchmarkMetricsResponse(KaggleObject): + r""" + Attributes: + benchmark_metric_producer_run_id (int) + """ + + def __init__(self): + self._benchmark_metric_producer_run_id = 0 + self._freeze() + + @property + def benchmark_metric_producer_run_id(self) -> int: + return self._benchmark_metric_producer_run_id + + @benchmark_metric_producer_run_id.setter + def benchmark_metric_producer_run_id(self, benchmark_metric_producer_run_id: int): + if benchmark_metric_producer_run_id is None: + del self.benchmark_metric_producer_run_id + return + if not isinstance(benchmark_metric_producer_run_id, int): + raise TypeError('benchmark_metric_producer_run_id must be of type int') + self._benchmark_metric_producer_run_id = benchmark_metric_producer_run_id + + @property + def benchmarkMetricProducerRunId(self): + return self.benchmark_metric_producer_run_id + + class ApiBenchmarkLeaderboard(KaggleObject): r""" Attributes: @@ -718,6 +839,192 @@ def body_fields(): return '*' +class ApiCreateBenchmarkMetricProducerRequest(KaggleObject): + r""" + Attributes: + owner_slug (str) + benchmark_slug (str) + version_number (int) + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + definition_source (DefinitionSource) + DatasetVersionSlug + optional SubPath where the script lives. + entrypoint (str) + Relative to definition_source.sub_path, executed via: cd && + python3 + input_options (BenchmarkMetricProducerInputOptions) + env_variables (EnvVariable) + secrets (UserSecret) + mounts (DatasetMount) + Optional. Additional Kaggle datasets mounted read-only into the producer + session container (e.g. rubrics or reference data). + """ + + def __init__(self): + self._owner_slug = "" + self._benchmark_slug = "" + self._version_number = None + self._definition_source = None + self._entrypoint = "" + self._input_options = None + self._env_variables = [] + self._secrets = [] + self._mounts = [] + self._freeze() + + @property + def owner_slug(self) -> str: + return self._owner_slug + + @owner_slug.setter + def owner_slug(self, owner_slug: str): + if owner_slug is None: + del self.owner_slug + return + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug + + @property + def benchmark_slug(self) -> str: + return self._benchmark_slug + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def definition_source(self) -> Optional['DefinitionSource']: + """DatasetVersionSlug + optional SubPath where the script lives.""" + return self._definition_source + + @definition_source.setter + def definition_source(self, definition_source: Optional['DefinitionSource']): + if definition_source is None: + del self.definition_source + return + if not isinstance(definition_source, DefinitionSource): + raise TypeError('definition_source must be of type DefinitionSource') + self._definition_source = definition_source + + @property + def entrypoint(self) -> str: + r""" + Relative to definition_source.sub_path, executed via: cd && + python3 + """ + return self._entrypoint + + @entrypoint.setter + def entrypoint(self, entrypoint: str): + if entrypoint is None: + del self.entrypoint + return + if not isinstance(entrypoint, str): + raise TypeError('entrypoint must be of type str') + self._entrypoint = entrypoint + + @property + def input_options(self) -> Optional['BenchmarkMetricProducerInputOptions']: + return self._input_options or None + + @input_options.setter + def input_options(self, input_options: Optional[Optional['BenchmarkMetricProducerInputOptions']]): + if input_options is None: + del self.input_options + return + if not isinstance(input_options, BenchmarkMetricProducerInputOptions): + raise TypeError('input_options must be of type BenchmarkMetricProducerInputOptions') + self._input_options = input_options + + @property + def env_variables(self) -> Optional[List[Optional['EnvVariable']]]: + return self._env_variables + + @env_variables.setter + def env_variables(self, env_variables: Optional[List[Optional['EnvVariable']]]): + if env_variables is None: + del self.env_variables + return + if not isinstance(env_variables, list): + raise TypeError('env_variables must be of type list') + if not all([isinstance(t, EnvVariable) for t in env_variables]): + raise TypeError('env_variables must contain only items of type EnvVariable') + self._env_variables = env_variables + + @property + def secrets(self) -> Optional[List[Optional['UserSecret']]]: + return self._secrets + + @secrets.setter + def secrets(self, secrets: Optional[List[Optional['UserSecret']]]): + if secrets is None: + del self.secrets + return + if not isinstance(secrets, list): + raise TypeError('secrets must be of type list') + if not all([isinstance(t, UserSecret) for t in secrets]): + raise TypeError('secrets must contain only items of type UserSecret') + self._secrets = secrets + + @property + def mounts(self) -> Optional[List[Optional['DatasetMount']]]: + r""" + Optional. Additional Kaggle datasets mounted read-only into the producer + session container (e.g. rubrics or reference data). + """ + return self._mounts + + @mounts.setter + def mounts(self, mounts: Optional[List[Optional['DatasetMount']]]): + if mounts is None: + del self.mounts + return + if not isinstance(mounts, list): + raise TypeError('mounts must be of type list') + if not all([isinstance(t, DatasetMount) for t in mounts]): + raise TypeError('mounts must contain only items of type DatasetMount') + self._mounts = mounts + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/versions/{version_number}/metric-producers/create' + else: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/metric-producers/create' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + class ApiCreateBenchmarkModelVersionConfigRequest(KaggleObject): r""" Attributes: @@ -1989,6 +2296,44 @@ def body_fields(): return '*' +class ApiUpdateBenchmarkArenaCombinationsResponse(KaggleObject): + r""" + Attributes: + not_added (BenchmarkArenaCombination) + Added combinations not saved because voters would have nothing to see: + the task names no artifact, or the candidate's counted run didn't + complete with output holding it. + """ + + def __init__(self): + self._not_added = [] + self._freeze() + + @property + def not_added(self) -> Optional[List[Optional['BenchmarkArenaCombination']]]: + r""" + Added combinations not saved because voters would have nothing to see: + the task names no artifact, or the candidate's counted run didn't + complete with output holding it. + """ + return self._not_added + + @not_added.setter + def not_added(self, not_added: Optional[List[Optional['BenchmarkArenaCombination']]]): + if not_added is None: + del self.not_added + return + if not isinstance(not_added, list): + raise TypeError('not_added must be of type list') + if not all([isinstance(t, BenchmarkArenaCombination) for t in not_added]): + raise TypeError('not_added must contain only items of type BenchmarkArenaCombination') + self._not_added = not_added + + @property + def notAdded(self): + return self.not_added + + class ApiUpdateBenchmarkArenaSettingsRequest(KaggleObject): r""" Attributes: @@ -2095,6 +2440,146 @@ def body_fields(): return '*' +class ApiValidateBenchmarkMetricProducerOutputRequest(KaggleObject): + r""" + Attributes: + owner_slug (str) + benchmark_slug (str) + version_number (int) + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + output_json (str) + Raw JSON text of a BenchmarkMetricProducerOutput file. + """ + + def __init__(self): + self._owner_slug = "" + self._benchmark_slug = "" + self._version_number = None + self._output_json = "" + self._freeze() + + @property + def owner_slug(self) -> str: + return self._owner_slug + + @owner_slug.setter + def owner_slug(self, owner_slug: str): + if owner_slug is None: + del self.owner_slug + return + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug + + @property + def benchmark_slug(self) -> str: + return self._benchmark_slug + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def output_json(self) -> str: + """Raw JSON text of a BenchmarkMetricProducerOutput file.""" + return self._output_json + + @output_json.setter + def output_json(self, output_json: str): + if output_json is None: + del self.output_json + return + if not isinstance(output_json, str): + raise TypeError('output_json must be of type str') + self._output_json = output_json + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/versions/{version_number}/metric-producers/validate-output' + else: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/metric-producers/validate-output' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + +class ApiValidateBenchmarkMetricProducerOutputResponse(KaggleObject): + r""" + Attributes: + is_valid (bool) + errors (str) + Validation errors found in output_json; empty when is_valid is true. + """ + + def __init__(self): + self._is_valid = False + self._errors = [] + self._freeze() + + @property + def is_valid(self) -> bool: + return self._is_valid + + @is_valid.setter + def is_valid(self, is_valid: bool): + if is_valid is None: + del self.is_valid + return + if not isinstance(is_valid, bool): + raise TypeError('is_valid must be of type bool') + self._is_valid = is_valid + + @property + def errors(self) -> Optional[List[str]]: + """Validation errors found in output_json; empty when is_valid is true.""" + return self._errors + + @errors.setter + def errors(self, errors: Optional[List[str]]): + if errors is None: + del self.errors + return + if not isinstance(errors, list): + raise TypeError('errors must be of type list') + if not all([isinstance(t, str) for t in errors]): + raise TypeError('errors must contain only items of type str') + self._errors = errors + + @property + def isValid(self): + return self.is_valid + + ApiAddTasksToBenchmarkRequest._fields = [ FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), @@ -2106,6 +2591,17 @@ def body_fields(): FieldMetadata("skippedTasks", "skipped_tasks", "_skipped_tasks", ApiBenchmarkTaskSlug, [], ListSerializer(KaggleObjectSerializer())), ] +ApiBatchScheduleBenchmarkMetricsRequest._fields = [ + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("benchmarkMetricProducerId", "benchmark_metric_producer_id", "_benchmark_metric_producer_id", int, 0, PredefinedSerializer()), +] + +ApiBatchScheduleBenchmarkMetricsResponse._fields = [ + FieldMetadata("benchmarkMetricProducerRunId", "benchmark_metric_producer_run_id", "_benchmark_metric_producer_run_id", int, 0, PredefinedSerializer()), +] + ApiBenchmarkLeaderboard.LeaderboardRow._fields = [ FieldMetadata("modelVersionName", "model_version_name", "_model_version_name", str, "", PredefinedSerializer()), FieldMetadata("modelVersionSlug", "model_version_slug", "_model_version_slug", str, "", PredefinedSerializer()), @@ -2153,6 +2649,18 @@ def body_fields(): FieldMetadata("settings", "settings", "_settings", BenchmarkArenaSettings, None, KaggleObjectSerializer()), ] +ApiCreateBenchmarkMetricProducerRequest._fields = [ + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("definitionSource", "definition_source", "_definition_source", DefinitionSource, None, KaggleObjectSerializer()), + FieldMetadata("entrypoint", "entrypoint", "_entrypoint", str, "", PredefinedSerializer()), + FieldMetadata("inputOptions", "input_options", "_input_options", BenchmarkMetricProducerInputOptions, None, KaggleObjectSerializer(), optional=True), + FieldMetadata("envVariables", "env_variables", "_env_variables", EnvVariable, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("secrets", "secrets", "_secrets", UserSecret, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("mounts", "mounts", "_mounts", DatasetMount, [], ListSerializer(KaggleObjectSerializer())), +] + ApiCreateBenchmarkModelVersionConfigRequest._fields = [ FieldMetadata("config", "config", "_config", ApiBenchmarkModelVersionConfig, None, KaggleObjectSerializer()), ] @@ -2268,6 +2776,10 @@ def body_fields(): FieldMetadata("remove", "remove", "_remove", BenchmarkArenaCombination, [], ListSerializer(KaggleObjectSerializer())), ] +ApiUpdateBenchmarkArenaCombinationsResponse._fields = [ + FieldMetadata("notAdded", "not_added", "_not_added", BenchmarkArenaCombination, [], ListSerializer(KaggleObjectSerializer())), +] + ApiUpdateBenchmarkArenaSettingsRequest._fields = [ FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), @@ -2276,3 +2788,15 @@ def body_fields(): FieldMetadata("updateMask", "update_mask", "_update_mask", FieldMask, None, FieldMaskSerializer()), ] +ApiValidateBenchmarkMetricProducerOutputRequest._fields = [ + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("outputJson", "output_json", "_output_json", str, "", PredefinedSerializer()), +] + +ApiValidateBenchmarkMetricProducerOutputResponse._fields = [ + FieldMetadata("isValid", "is_valid", "_is_valid", bool, False, PredefinedSerializer()), + FieldMetadata("errors", "errors", "_errors", str, [], ListSerializer(PredefinedSerializer())), +] + diff --git a/kagglesdk/kernels/types/kernels_api_service.py b/kagglesdk/kernels/types/kernels_api_service.py index 1ca8324..9964d72 100644 --- a/kagglesdk/kernels/types/kernels_api_service.py +++ b/kagglesdk/kernels/types/kernels_api_service.py @@ -191,7 +191,6 @@ class ApiCreateKernelSessionRequest(KaggleObject): machine_shape (str) The machine shape to use for this session. Currently supported options: * NvidiaTeslaT4 - * NvidiaTeslaP100 * Tpu1VmV38 enable_internet (bool) Whether or not the kernel should be able to access the internet in this @@ -279,7 +278,6 @@ def machine_shape(self) -> str: r""" The machine shape to use for this session. Currently supported options: * NvidiaTeslaT4 - * NvidiaTeslaP100 * Tpu1VmV38 """ return self._machine_shape or "" @@ -2028,7 +2026,6 @@ class ApiSaveKernelRequest(KaggleObject): machine_shape (str) The machine shape to use for this session. Currently supported options: * NvidiaTeslaT4 - * NvidiaTeslaP100 * Tpu1VmV38 """ @@ -2393,7 +2390,6 @@ def machine_shape(self) -> str: r""" The machine shape to use for this session. Currently supported options: * NvidiaTeslaT4 - * NvidiaTeslaP100 * Tpu1VmV38 """ return self._machine_shape or ""