Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion crates/vt/src/session/cache/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -803,7 +803,13 @@ impl ExecutionCache {
let start = Instant::now();
let fetched = self
.remote_clients
.fetch(remote_config, cache_key, &cache_metadata.execution_cache_key, cancel_token)
.fetch(
remote_config,
cache_metadata.platform,
cache_key,
&cache_metadata.execution_cache_key,
cancel_token,
)
.await;
let (outcome, entries) = match &fetched {
Ok(Some(Fetched::Exact { entries })) => ("exact", entries.len()),
Expand All @@ -820,9 +826,11 @@ impl ExecutionCache {
);
let (cache_key, validation, dir) =
(cache_key.clone(), validation.clone(), self.dir.clone());
let platform = cache_metadata.platform;
let check = move || {
remote::resolve(
fetched,
platform,
&cache_key,
|cache_value| validation.validate(cache_value),
|cache_value| {
Expand Down Expand Up @@ -1008,6 +1016,7 @@ impl ExecutionCache {
};
match self.remote_clients.prepare_upload(
remote_config,
cache_metadata.platform,
&cache_key,
execution_cache_key,
&cache_value,
Expand Down
163 changes: 130 additions & 33 deletions crates/vt/src/session/cache/remote.rs

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions crates/vt_bin/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ impl vt::CommandHandler for CommandHandler {
input: None,
output: None,
remote: None,
platform: None,
}),
envs: Arc::clone(&command.envs),
}))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,91 @@ steps = [
],
]

# `generate` sets `cache.platform: []`, so its remote keys name no OS or
# architecture. Only one platform runs here, so this case checks that its entry
# round trips and that the setting changes the remote keys but not the local
# cache.
[[e2e]]
name = "platform_independent"
cfg = 'not(target_env = "musl")'
ignore = true
steps = [
[
"remote-cache-server",
"start",
],
{ argv = [
"remote-cache-server",
"run",
"--github-actions",
"vt",
"run",
"generate",
], envs = [
[
"VP_REMOTE_CACHE",
"read-write",
],
[
"VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS",
"1",
],
] },
[
"vt",
"cache",
"clean",
],
[
"vtt",
"rm",
"-rf",
"dist",
],
{ argv = [
"remote-cache-server",
"run",
"vt",
"run",
"generate",
], comment = "The entry stored under the keys that name no platform is a remote hit." },
{ argv = [
"vtt",
"print-file",
"dist/generated.txt",
], comment = "The outputs are restored." },
[
"vtt",
"replace-file-content",
"vite-task.json",
'"platform": []',
'"platform": ["os", "arch"]',
],
{ argv = [
"remote-cache-server",
"run",
"vt",
"run",
"generate",
], comment = "The local cache doesn't depend on the setting, so this is a local hit with no requests." },
[
"vt",
"cache",
"clean",
],
{ argv = [
"remote-cache-server",
"run",
"vt",
"run",
"generate",
], comment = "The remote keys name this platform now, so neither matches the stored entry's keys." },
[
"remote-cache-server",
"stop",
],
]

[[e2e]]
name = "corrupt_archive"
cfg = 'not(target_env = "musl")'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# platform_independent

## `remote-cache-server start`

```
```

## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run generate`

```
$ vtt write-file dist/generated.txt generated

[remote-cache] POST /fetch 404
[remote-cache] POST /store 200
```

## `vt cache clean`

```
```

## `vtt rm -rf dist`

```
```

## `remote-cache-server run vt run generate`

The entry stored under the keys that name no platform is a remote hit.

```
$ vtt write-file dist/generated.txt generated ◉ remote cache hit, replaying

---
vt run: remote cache hit.
[remote-cache] POST /fetch 200 exact [1]
[remote-cache] GET /blob/1 200
```

## `vtt print-file dist/generated.txt`

The outputs are restored.

```
generated
```

## `vtt replace-file-content vite-task.json '"platform": []' '"platform": ["os", "arch"]'`

```
```

## `remote-cache-server run vt run generate`

The local cache doesn't depend on the setting, so this is a local hit with no requests.

```
$ vtt write-file dist/generated.txt generated ◉ cache hit, replaying

---
vt run: cache hit.
```

## `vt cache clean`

```
```

## `remote-cache-server run vt run generate`

The remote keys name this platform now, so neither matches the stored entry's keys.

```
$ vtt write-file dist/generated.txt generated

[remote-cache] POST /fetch 404
```

## `remote-cache-server stop`

```
```
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,14 @@
"command": "vtt exit 1",
"dependsOn": ["build"],
"cache": false
},
"generate": {
"command": "vtt write-file dist/generated.txt generated",
"cache": {
"input": ["src/**"],
"output": ["dist/**"],
"platform": []
}
}
}
}
43 changes: 40 additions & 3 deletions crates/vt_graph/src/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ use hashbrown::hash_map::EntryRef;
use rustc_hash::{FxBuildHasher, FxHashSet};
use serde::Serialize;
pub use user::{
AutoTracking, Command, EnabledCacheConfig, GlobWithBase, InputBase, ResolvedGlobalCacheConfig,
UserCacheConfig, UserDependencyType, UserDependsOnEntry, UserDependsOnFrom,
UserGlobalCacheConfig, UserInputEntry, UserInputsConfig, UserOutputEntry,
AutoTracking, Command, EnabledCacheConfig, GlobWithBase, InputBase, PlatformDimension,
ResolvedGlobalCacheConfig, UserCacheConfig, UserDependencyType, UserDependsOnEntry,
UserDependsOnFrom, UserGlobalCacheConfig, UserInputEntry, UserInputsConfig, UserOutputEntry,
UserPackageDependency, UserRunConfig, UserTaskConfig, UserTaskDefinition,
};
use vt_path::AbsolutePath;
Expand Down Expand Up @@ -87,6 +87,10 @@ impl ResolvedTaskOptions {

Some(CacheConfig {
remote_cache_allowed: enabled_cache_config.remote.unwrap_or(true),
platform: enabled_cache_config
.platform
.as_deref()
.map(CachePlatform::from_dimensions),
env_config: EnvConfig {
fingerprinted_envs: enabled_cache_config
.env
Expand All @@ -108,11 +112,44 @@ pub struct CacheConfig {
/// Whether the task's `cache.remote` allows remote caching. An execution uses
/// the remote cache only if its `vp run` level also resolves one.
pub remote_cache_allowed: bool,
/// The task's `cache.platform`, or `None` if it doesn't set one. Planning
/// replaces `None` with the workspace's `cache.platform`.
pub platform: Option<CachePlatform>,
pub env_config: EnvConfig,
pub input_config: ResolvedGlobConfig,
pub output_config: ResolvedGlobConfig,
}

/// The parts of the platform that a task's cache entries depend on. Remote
/// cache entries are shared only between machines whose listed parts match.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct CachePlatform {
/// Whether the entries depend on the operating system.
pub os: bool,
/// Whether the entries depend on the CPU architecture.
pub arch: bool,
}

impl CachePlatform {
/// Every part of the platform: the default.
pub const ALL: Self = Self { os: true, arch: true };

/// The parts that `dimensions` lists.
#[must_use]
pub fn from_dimensions(dimensions: &[PlatformDimension]) -> Self {
Self {
os: dimensions.contains(&PlatformDimension::Os),
arch: dimensions.contains(&PlatformDimension::Arch),
}
}

/// The parts that either `self` or `other` lists.
#[must_use]
pub const fn union(self, other: Self) -> Self {
Self { os: self.os || other.os, arch: self.arch || other.arch }
}
}

/// Resolved input configuration for cache fingerprinting.
///
/// This is the normalized form after parsing user config.
Expand Down
Loading
Loading