diff --git a/crates/socket-patch-cli/tests/e2e_safety_yarn_pnp.rs b/crates/socket-patch-cli/tests/e2e_safety_yarn_pnp.rs index 74461b588..e54c3e9ef 100644 --- a/crates/socket-patch-cli/tests/e2e_safety_yarn_pnp.rs +++ b/crates/socket-patch-cli/tests/e2e_safety_yarn_pnp.rs @@ -1304,6 +1304,68 @@ fn stale_pnp_loader_under_yaml_spelled_linker_applies() { } } +/// #1611: yarn 4 expands `${NAME}` in `.yarnrc.yml` with the variables of +/// the project's `.env.yarn` applied, a file value winning over an +/// exported one. A live loader whose linker `.env.yarn` turns into `pnp` +/// is refused instead of exiting 0 `package_not_installed`, even when the +/// shell exports a non-PnP value for the same variable. +#[test] +fn pnp_loader_under_env_file_pnp_linker_refuses() { + for env in [&[][..], &[("SOCKET_PATCH_T1611_L", "node-modules")][..]] { + let dir = tempfile::tempdir().unwrap(); + make_yarn_berry_project(dir.path()); + std::fs::write( + dir.path().join(".yarnrc.yml"), + "nodeLinker: \"${SOCKET_PATCH_T1611_L:-node-modules}\"\n", + ) + .unwrap(); + std::fs::write(dir.path().join(".env.yarn"), "SOCKET_PATCH_T1611_L=pnp\n").unwrap(); + let index = stage_applicable_package(dir.path()); + let (code, stdout, _) = run_with_env(dir.path(), &["apply", "--json"], env); + assert_eq!(code, 1, "{env:?}: {stdout}"); + let envelope = parse_json_envelope(&stdout); + assert_eq!( + envelope_error_code(&envelope), + Some("yarn_pnp_unsupported"), + "{env:?}" + ); + assert_eq!(std::fs::read(&index).unwrap(), ORIGINAL_BYTES, "{env:?}"); + } +} + +/// #1611: the other direction. `.env.yarn` turns `${NAME:-pnp}` into +/// `node-modules`, so a stale loader is disowned and apply patches the +/// installed copy. +#[test] +fn stale_pnp_loader_under_env_file_linker_applies() { + let dir = tempfile::tempdir().unwrap(); + let cwd = dir.path(); + std::fs::write( + cwd.join("package.json"), + r#"{"name":"migrated","version":"0.0.0","private":true}"#, + ) + .unwrap(); + std::fs::write(cwd.join("yarn.lock"), "__metadata:\n version: 8\n").unwrap(); + std::fs::write(cwd.join(".pnp.js"), b"// stale Yarn 2 loader\n").unwrap(); + std::fs::write( + cwd.join(".yarnrc.yml"), + "nodeLinker: \"${SOCKET_PATCH_T1611_L:-pnp}\"\n", + ) + .unwrap(); + std::fs::write( + cwd.join(".env.yarn"), + "# yarn injects this\nexport SOCKET_PATCH_T1611_L='node-modules'\n", + ) + .unwrap(); + let index = stage_applicable_package(cwd); + let (code, stdout, stderr) = run(cwd, &["apply", "--json"]); + assert_eq!( + code, 0, + "a stale loader must not refuse.\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + assert_eq!(std::fs::read(&index).unwrap(), PATCHED_BYTES); +} + /// #1129: pnpm's `node-linker=pnp` with a custom `modulesDir` keeps its /// store in `/.pnpm`, not `node_modules/.pnpm`. The layout /// detector reads the same modules dirs as the crawler, so the tree is diff --git a/crates/socket-patch-cli/tests/in_process_vendor.rs b/crates/socket-patch-cli/tests/in_process_vendor.rs index 92e5747a6..a95289285 100644 --- a/crates/socket-patch-cli/tests/in_process_vendor.rs +++ b/crates/socket-patch-cli/tests/in_process_vendor.rs @@ -1520,6 +1520,51 @@ async fn berry_lock_only_pnp_project_refused_up_front() { } } +/// #1611: yarn 4 loads the project's `injectEnvironmentFiles` (`.env.yarn` +/// by default) before it expands `${NAME}` in `.yarnrc.yml`, so a linker +/// that a `.env.yarn` variable turns into `node-modules` is not PnP: +/// vendor wires the project instead of refusing it. An emptied +/// `injectEnvironmentFiles` loads nothing, and the fallback (`pnp`) is +/// refused as before. +#[tokio::test] +async fn berry_env_file_linker_decides_vendoring() { + const RC: &str = "nodeLinker: \"${SOCKET_PATCH_T1611_VL:-pnp}\"\nenableGlobalCache: false\n"; + for (label, rc_extra, file, vendors) in [ + ("default .env.yarn", "", ".env.yarn", true), + ( + "custom file", + "injectEnvironmentFiles: [\".env.custom\"]\n", + ".env.custom", + true, + ), + ( + "emptied list", + "injectEnvironmentFiles: []\n", + ".env.yarn", + false, + ), + ] { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + stage_berry_project(root, BERRY_WIN_PKG, &berry_win_lock()); + std::fs::write(root.join(".yarnrc.yml"), format!("{RC}{rc_extra}")).unwrap(); + std::fs::write(root.join(file), "SOCKET_PATCH_T1611_VL=node-modules\n").unwrap(); + let (code, env) = vendor_cli(root, &[]); + let wired = std::fs::read_to_string(root.join("yarn.lock")) + .unwrap() + .contains(".socket/vendor/npm/"); + if vendors { + assert_eq!(code, 0, "{label}: {env:#}"); + assert_eq!(env["summary"]["applied"], 1, "{label}: {env:#}"); + assert!(wired, "{label}: wired"); + } else { + assert_eq!(code, 1, "{label}: {env:#}"); + find_event(&env, "failed", Some("vendor_yarn_berry_unsupported")); + assert!(!wired, "{label}: not wired"); + } + } +} + /// Mount the hosted-mode API (batch discovery, per-package search, a granted /// reference carrying the yarn-berry-zip checksum, the patch view) for the /// berry takeover legs. Returns the hosted tarball URL. diff --git a/crates/socket-patch-core/src/crawlers/pkg_managers.rs b/crates/socket-patch-core/src/crawlers/pkg_managers.rs index 794253435..dae80740e 100644 --- a/crates/socket-patch-core/src/crawlers/pkg_managers.rs +++ b/crates/socket-patch-core/src/crawlers/pkg_managers.rs @@ -176,6 +176,20 @@ pub(crate) struct YarnEnv { /// The variables `${NAME}` references read; `None` reads this /// process's environment. pub vars: Option>, + /// The variables yarn 4 injects from the project's + /// `injectEnvironmentFiles` ([`Self::with_env_files`]): they win over + /// `vars`, as yarn applies them over its environment (#1611). + pub env_files: EnvFiles, +} + +/// The dotenv files a yarn run injects, read on the first `${NAME}` +/// lookup: most runs expand none. +#[derive(Debug, Clone, Default)] +pub(crate) struct EnvFiles { + /// The folder yarn runs in and `YARN_INJECT_ENVIRONMENT_FILES`; `None` + /// injects nothing. + run: Option<(PathBuf, Option)>, + vars: std::sync::OnceLock>, } impl YarnEnv { @@ -189,9 +203,58 @@ impl YarnEnv { .unwrap_or_else(|| ".yarnrc.yml".to_string()), home: crate::utils::fs::home_dir(), vars: None, + env_files: Default::default(), } } + /// [`Self::current`] for a yarn run in `project_root`: with the + /// variables of the project's `injectEnvironmentFiles` (#1611). + pub(crate) fn for_project(project_root: &Path) -> Self { + let env_list = yarn_env_key(std::env::vars_os(), "YARN_INJECT_ENVIRONMENT_FILES", true); + Self::current().with_env_files(project_root, env_list.as_deref()) + } + + /// This environment with the variables yarn 4 injects for a run in + /// `project_root`: the dotenv files `injectEnvironmentFiles` names + /// (`env_list` when the environment sets it, else the nearest rc file + /// at or above the project that sets it, else the home rc file's, + /// else `.env.yarn`), a later file winning. + pub(crate) fn with_env_files(mut self, project_root: &Path, env_list: Option<&str>) -> Self { + let start = + std::path::absolute(project_root).unwrap_or_else(|_| project_root.to_path_buf()); + self.env_files = EnvFiles { + run: Some((start, env_list.map(str::to_string))), + vars: Default::default(), + }; + self + } + + /// The variables the env files inject, read once. + fn injected(&self) -> &std::collections::BTreeMap { + self.env_files.vars.get_or_init(|| { + let Some((start, env_list)) = &self.env_files.run else { + return Default::default(); + }; + let read = |path: &Path| crate::utils::fs::read_regular_to_string_sync(path).ok(); + let mut rcs: Vec<(PathBuf, String)> = start + .ancestors() + .filter_map(|dir| Some((dir.to_path_buf(), read(&dir.join(&self.rc_filename))?))) + .collect(); + if let Some(home) = &self.home { + if !rcs.iter().any(|(dir, _)| dir == home) { + if let Some(rc) = read(&home.join(&self.rc_filename)) { + rcs.push((home.clone(), rc)); + } + } + } + let rcs: Vec<(&Path, &str)> = + rcs.iter().map(|(d, t)| (d.as_path(), t.as_str())).collect(); + let base_var = |name: &str| self.base_var(name); + let paths = crate::formats::yarn::rc::env_file_paths(env_list.as_deref(), &rcs, start); + crate::formats::yarn::rc::injected_env(&paths, &read, &base_var) + }) + } + /// The repository alone, not the host it is scanned on: no /// environment override, no home rc file, and no variables, so a /// `${NAME}` reference takes its fallback or fails closed. For a view @@ -202,10 +265,21 @@ impl YarnEnv { rc_filename: ".yarnrc.yml".to_string(), home: None, vars: Some(Default::default()), + env_files: Default::default(), } } - fn var(&self, name: &str) -> Option { + /// The value yarn expands `${name}` to: an injected variable, else + /// one of `vars`. + pub(crate) fn var(&self, name: &str) -> Option { + if let Some(value) = self.injected().get(name) { + return Some(value.clone()); + } + self.base_var(name) + } + + /// A variable of `vars`, or of this process's environment. + fn base_var(&self, name: &str) -> Option { match &self.vars { Some(vars) => vars.get(name).cloned(), None => std::env::var(name).ok(), @@ -274,7 +348,7 @@ pub(crate) fn rc_node_linker(rc: &str, env: &YarnEnv) -> Option { /// is `YARN_RC_FILENAME` when set. `None` when nothing sets it: yarn berry /// then uses its default linker, `pnp`. pub fn yarn_node_linker(project_root: &Path) -> Option { - yarn_node_linker_in(project_root, &YarnEnv::current()) + yarn_node_linker_in(project_root, &YarnEnv::for_project(project_root)) } pub(crate) fn yarn_node_linker_in(project_root: &Path, env: &YarnEnv) -> Option { @@ -708,9 +782,98 @@ mod tests { rc_filename: ".yarnrc.yml".into(), home: None, vars: Some(Default::default()), + env_files: Default::default(), } } + /// #1611: yarn 4 expands `${…}` with `injectEnvironmentFiles` applied + /// (`.env.yarn` by default), a file's value winning over an exported + /// one; an rc file or `YARN_INJECT_ENVIRONMENT_FILES` renames or + /// empties the list, and a later file wins over an earlier one. + #[test] + fn env_files_feed_rc_references() { + let linker = |rc_extra: &str, files: &[(&str, &str)], env_list: Option<&str>| { + let d = tempfile::tempdir().unwrap(); + let rc = format!("nodeLinker: \"${{L:-pnp}}\"\n{rc_extra}"); + std::fs::write(d.path().join(".yarnrc.yml"), rc).unwrap(); + for (name, text) in files { + std::fs::write(d.path().join(name), text).unwrap(); + } + let mut env = bare_env(None); + env.vars = Some([("L".to_string(), "pnpm".to_string())].into()); + let env = env.with_env_files(d.path(), env_list); + yarn_node_linker_in(d.path(), &env) + }; + let nm = Some("node-modules".to_string()); + // No file: the exported variable. + assert_eq!(linker("", &[], None).as_deref(), Some("pnpm")); + // `.env.yarn` overrides it. + assert_eq!(linker("", &[(".env.yarn", "L=node-modules\n")], None), nm); + // An rc list replaces the default; `[]` loads nothing. + let custom = [(".env.yarn", "L=pnp\n"), (".env.ci", "L=node-modules\n")]; + assert_eq!( + linker("injectEnvironmentFiles: [.env.ci]\n", &custom, None), + nm + ); + assert_eq!( + linker("injectEnvironmentFiles: []\n", &custom, None).as_deref(), + Some("pnpm") + ); + // A later file wins; a missing optional one is skipped. + assert_eq!( + linker( + "injectEnvironmentFiles:\n - .env.yarn\n - .env.missing?\n - .env.ci\n", + &custom, + None + ), + nm + ); + // The environment's list beats the rc file's. + assert_eq!( + linker("injectEnvironmentFiles: []\n", &custom, Some(".env.ci")), + nm + ); + } + + /// #1611: an rc file's relative `injectEnvironmentFiles` entries + /// resolve against its own folder, and every rc file's list loads, the + /// nearest last. + #[test] + fn env_files_resolve_against_their_rc_folder() { + let w = tempfile::tempdir().unwrap(); + std::fs::write( + w.path().join(".yarnrc.yml"), + "nodeLinker: \"${L:-pnp}\"\ninjectEnvironmentFiles: [env/linker]\n", + ) + .unwrap(); + std::fs::create_dir_all(w.path().join("env")).unwrap(); + std::fs::write(w.path().join("env/linker"), "L=node-modules\n").unwrap(); + let proj = w.path().join("packages/app"); + std::fs::create_dir_all(&proj).unwrap(); + let env = bare_env(None).with_env_files(&proj, None); + assert_eq!( + yarn_node_linker_in(&proj, &env).as_deref(), + Some("node-modules") + ); + // Yarn concatenates every rc file's list: a nearer `[]` clears + // nothing (yarn 4.9.1 resolves `node-modules` here)... + std::fs::write(proj.join(".yarnrc.yml"), "injectEnvironmentFiles: []\n").unwrap(); + let env = bare_env(None).with_env_files(&proj, None); + assert_eq!( + yarn_node_linker_in(&proj, &env).as_deref(), + Some("node-modules") + ); + // ...and a nearer file loads last, so its value wins. + std::fs::write( + proj.join(".yarnrc.yml"), + "injectEnvironmentFiles: [.env.app]\n", + ) + .unwrap(); + std::fs::write(proj.join(".env.app"), "L=pnpm\n").unwrap(); + let env = bare_env(None).with_env_files(&proj, None); + assert_eq!(yarn_node_linker_in(&proj, &env).as_deref(), Some("pnpm")); + } + /// yarn reads the home folder's rc file after every project-side one, /// and `YARN_RC_FILENAME` renames the rc file it looks for everywhere. #[test] diff --git a/crates/socket-patch-core/src/formats/yarn/rc.rs b/crates/socket-patch-core/src/formats/yarn/rc.rs index abf58e52a..5ecc25900 100644 --- a/crates/socket-patch-core/src/formats/yarn/rc.rs +++ b/crates/socket-patch-core/src/formats/yarn/rc.rs @@ -11,8 +11,14 @@ //! Every `.yarnrc.yml` scalar socket-patch decides on (`nodeLinker`, //! `compressionLevel`, `pnpmStoreFolder`) is read through [`setting`], so //! no reader takes the literal `${…}` text for the value yarn uses. +//! +//! Yarn 4 expands those references with the variables of its +//! `injectEnvironmentFiles` (`.env.yarn` by default) applied, a file's +//! value winning over an exported one (#1611): [`env_file_paths`] names +//! the files and [`injected_env`] reads them. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; use serde_saphyr::granit_parser::{Event, Options, Parser}; @@ -39,12 +45,6 @@ pub fn setting(rc: &str, key: &str, var: &dyn Fn(&str) -> Option) -> Opt Some(YarnrcSetting { raw, value }) } -/// [`setting`] with the references expanded from this process's -/// environment. -pub fn process_setting(rc: &str, key: &str) -> Option { - setting(rc, key, &|name| std::env::var(name).ok()) -} - /// The top-level scalar `key` of `rc` as written (quotes removed, nothing /// expanded), or `None` when it is absent or not a scalar. The file is /// parsed as YAML after one leading BOM (yarn's parser skips one); a file @@ -57,18 +57,85 @@ pub fn raw_setting(rc: &str, key: &str) -> Option { if strip_bom(rc).len() < rc.len() { return line_scan(rc, key); } - match top_level_scalars(rc) { - Ok(mut settings) => settings.remove(key).flatten(), + match top_level_values(rc) { + Ok(mut settings) => match settings.remove(key)?.value { + RcValue::Scalar(value) => Some(value), + _ => None, + }, Err(()) => line_scan(rc, key), } } -/// The root mapping's keys, each with its scalar value (`None` for a -/// nested mapping, sequence or alias); the last occurrence of a key wins, +/// The top-level list setting `key` of `rc` as written (nothing +/// expanded): a sequence's scalar items, or a scalar split at its commas, +/// as yarn reads an array setting given as a string. The last occurrence +/// wins; `Some(vec![])` empties the list. `None` when the file does not +/// set `key` to a list or scalar, or yarn would refuse it (not a single +/// YAML document with a mapping root, or a second BOM). +pub fn raw_list_setting(rc: &str, key: &str) -> Option> { + list_setting(rc, key).map(|setting| setting.items) +} + +/// A list setting as one rc file sets it ([`list_setting`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListSetting { + pub items: Vec, + /// Given as a string, not a sequence: yarn takes it as is instead of + /// merging it with other rc files' lists. + pub scalar: bool, + /// Yarn's `onConflict` for the value (`{onConflict: reset, value: + /// [...]}`): `reset` keeps this list and drops lower-precedence ones, + /// `hardReset` drops this one too. + pub on_conflict: Option, +} + +/// [`raw_list_setting`] with how yarn merges it across rc files. +pub fn list_setting(rc: &str, key: &str) -> Option { + let rc = strip_bom(rc); + if strip_bom(rc).len() < rc.len() { + return None; + } + let setting = top_level_values(rc).ok()?.remove(key)?; + let (items, scalar) = match setting.value { + RcValue::Scalar(value) => (split_list(&value), true), + RcValue::List(items) => (items, false), + RcValue::Other => return None, + }; + Some(ListSetting { + items, + scalar, + on_conflict: setting.on_conflict, + }) +} + +/// A string given for an array setting, split at its commas as yarn +/// splits it. +fn split_list(value: &str) -> Vec { + // Yarn's `split(/,/)`: an item keeps its spaces (` .env.b` names a + // file with a leading space), and an empty one names the folder + // itself, which no dotenv read accepts. + value + .split(',') + .filter(|item| !item.is_empty()) + .map(str::to_string) + .collect() +} + +/// A top-level setting's value, as [`top_level_values`] reads it. +enum RcValue { + Scalar(String), + /// A sequence: its scalar items. + List(Vec), + /// A nested mapping or an alias. + Other, +} + +/// The root mapping's keys, each with its value (a sequence's scalar +/// items; nothing for a nested mapping or alias); the last occurrence of a key wins, /// as in yarn. A root that is not a mapping (a list, a lone scalar) sets /// nothing: yarn refuses such a file. `Err` when the text is not a single /// YAML document. -fn top_level_scalars(rc: &str) -> Result>, ()> { +fn top_level_values(rc: &str) -> Result, ()> { let mut options = Options::default(); options.emit_comments = false; let mut out = HashMap::new(); @@ -78,6 +145,11 @@ fn top_level_scalars(rc: &str) -> Result>, ()> { // Whether the next root-level node is a key (true) or a value. let mut at_key = true; let mut root_is_mapping = false; + // The setting whose sequence value is being read, with its items. + let mut list: Option<(String, Vec)> = None; + // The setting whose mapping value is being read: yarn's + // `{onConflict, value}` wrapper when it has a string `onConflict`. + let mut wrapped: Option = None; for event in Parser::new_from_str_with_options(rc, options) { let (event, _) = event.map_err(|_| ())?; match event { @@ -93,42 +165,296 @@ fn top_level_scalars(rc: &str) -> Result>, ()> { root_is_mapping = matches!(event, Event::MappingStart(..)); } else if depth == 1 && !at_key { if let Some(key) = pending_key.take() { - out.insert(key, None); + if !root_is_mapping { + } else if matches!(event, Event::SequenceStart(..)) { + list = Some((key, Vec::new())); + } else { + wrapped = Some(Wrapped::new(key)); + } } at_key = true; } else if depth == 1 { // A complex key: not a setting name. pending_key = None; at_key = false; + } else if let Some(w) = wrapped.as_mut() { + w.nested(depth, matches!(event, Event::SequenceStart(..))); } depth += 1; } - Event::MappingEnd | Event::SequenceEnd => depth = depth.saturating_sub(1), + Event::MappingEnd | Event::SequenceEnd => { + depth = depth.saturating_sub(1); + if depth == 1 { + if let Some((key, items)) = list.take() { + out.insert(key, RcSetting::plain(RcValue::List(items))); + } + if let Some(w) = wrapped.take() { + let (key, setting) = w.finish(); + out.insert(key, setting); + } + } else if depth == 2 { + if let Some(w) = wrapped.as_mut() { + w.end_nested(); + } + } + } Event::Scalar(text, ..) if depth == 1 && root_is_mapping => { if at_key { pending_key = Some(text.into_owned()); at_key = false; } else { if let Some(key) = pending_key.take() { - out.insert(key, Some(text.into_owned())); + out.insert(key, RcSetting::plain(RcValue::Scalar(text.into_owned()))); } at_key = true; } } + Event::Scalar(text, ..) if depth == 2 && list.is_some() => { + if let Some((_, items)) = list.as_mut() { + items.push(text.into_owned()); + } + } + Event::Scalar(text, ..) if depth >= 2 => { + if let Some(w) = wrapped.as_mut() { + w.scalar(depth, text.into_owned()); + } + } Event::Alias(_) if depth == 1 && root_is_mapping => { if !at_key { if let Some(key) = pending_key.take() { - out.insert(key, None); + out.insert(key, RcSetting::plain(RcValue::Other)); } } at_key = !at_key; } + Event::Alias(_) if depth == 2 => { + if let Some(w) = wrapped.as_mut() { + w.other_value(); + } + } _ => {} } } Ok(out) } +/// A top-level setting: its value and yarn's `onConflict` for it. +struct RcSetting { + value: RcValue, + on_conflict: Option, +} + +impl RcSetting { + fn plain(value: RcValue) -> Self { + Self { + value, + on_conflict: None, + } + } +} + +/// A mapping value being read: `{onConflict: reset, value: [...]}` is +/// the value with an `onConflict`, any other mapping is [`RcValue::Other`]. +struct Wrapped { + key: String, + on_conflict: Option, + value: Option, + /// The mapping key the next depth-2 node is the value of. + entry: Option, + /// The `value` entry's sequence items, while it is read. + items: Option>, +} + +impl Wrapped { + fn new(key: String) -> Self { + Self { + key, + on_conflict: None, + value: None, + entry: None, + items: None, + } + } + + /// A scalar at `depth` (the mapping's own entries sit at depth 2). + fn scalar(&mut self, depth: usize, text: String) { + if depth == 3 { + if let Some(items) = self.items.as_mut() { + items.push(text); + } + return; + } + match self.entry.take() { + None => self.entry = Some(text), + Some(entry) if entry == "onConflict" => self.on_conflict = Some(text), + Some(entry) if entry == "value" => self.value = Some(RcValue::Scalar(text)), + Some(_) => {} + } + } + + /// A collection opening at `depth`. + fn nested(&mut self, depth: usize, sequence: bool) { + if depth != 2 { + return; + } + match self.entry.take() { + Some(entry) if entry == "value" && sequence => self.items = Some(Vec::new()), + Some(entry) if entry == "value" => self.value = Some(RcValue::Other), + // A complex key, or another entry's collection. + _ => {} + } + } + + /// A depth-2 collection closing. + fn end_nested(&mut self) { + if let Some(items) = self.items.take() { + self.value = Some(RcValue::List(items)); + } + } + + fn other_value(&mut self) { + if self.entry.take().as_deref() == Some("value") { + self.value = Some(RcValue::Other); + } + } + + fn finish(self) -> (String, RcSetting) { + let setting = match (self.on_conflict, self.value) { + (Some(on_conflict), Some(value)) => RcSetting { + value, + on_conflict: Some(on_conflict), + }, + _ => RcSetting::plain(RcValue::Other), + }; + (self.key, setting) + } +} + +/// `injectEnvironmentFiles`' default: the folder yarn runs in's +/// `.env.yarn`, optional. +pub const DEFAULT_ENV_FILES: &[&str] = &[".env.yarn?"]; + +/// The environment files yarn 4 loads (`injectEnvironmentFiles`), in load +/// order. The list is `env_list` (`YARN_INJECT_ENVIRONMENT_FILES`, comma +/// separated, against `start`, the folder yarn runs in) when set. Else +/// yarn merges the lists of the rc files in `rcs` (highest precedence +/// first: the nearest folder's, each ancestor's, the home folder's; each +/// with the folder its relative entries resolve against), walking from +/// the highest: each sequence goes before those already taken, so the +/// lowest-precedence file's entries load first and a nearer `[]` clears +/// nothing. The walk stops after a list whose `onConflict` is `reset`, +/// before one whose `onConflict` is `hardReset` or that is a string while +/// the first was a sequence (or the reverse); a string from the first +/// file that sets the list is the whole list. When no rc file sets it, +/// the list is [`DEFAULT_ENV_FILES`] against `start`. A string's entries +/// resolve against `start` too. An entry's `${NAME}` references expand +/// with no variable set, as yarn reads the list before its environment; +/// one that cannot (yarn refuses to run) is skipped. The optional marker (`?`) is dropped: yarn stops on a missing +/// required file, so no run of it reads a value from one. +pub fn env_file_paths(env_list: Option<&str>, rcs: &[(&Path, &str)], start: &Path) -> Vec { + let entries: Vec<(&Path, String)> = match env_list { + Some(list) => split_list(list).into_iter().map(|e| (start, e)).collect(), + None => { + // yarn's `resolveValueAt` over the rc files. + let mut taken: Vec<(&Path, Vec)> = Vec::new(); + let mut first_scalar = None; + for (dir, rc) in rcs { + let Some(setting) = list_setting(rc, "injectEnvironmentFiles") else { + continue; + }; + let scalar = *first_scalar.get_or_insert(setting.scalar); + if setting.scalar != scalar || setting.on_conflict.as_deref() == Some("hardReset") { + break; + } + // A string's items carry no source file, so yarn resolves + // them against the folder it runs in. + taken.insert(0, (if scalar { start } else { *dir }, setting.items)); + if scalar || setting.on_conflict.as_deref() == Some("reset") { + break; + } + } + if first_scalar.is_none() { + DEFAULT_ENV_FILES + .iter() + .map(|f| (start, f.to_string())) + .collect() + } else { + taken + .into_iter() + .flat_map(|(dir, list)| list.into_iter().map(move |e| (dir, e))) + .collect() + } + } + }; + entries + .iter() + .filter_map(|(base, entry)| { + // Yarn reads this list before it loads any environment, so + // `${NAME}` sees no variable: its fallback or nothing. + let entry = expand_env(entry, &mut |_| Ok(None)).ok()?; + let path = entry.strip_suffix('?').unwrap_or(&entry); + (!path.is_empty()).then(|| base.join(path)) + }) + .collect() +} + +/// The variables yarn adds to its environment from the dotenv files +/// `paths` ([`env_file_paths`]), as `read` gives their text (`None`: the +/// file is missing or unreadable, which injects nothing). Each value has +/// its `${NAME}` references expanded, as yarn does, against the variables +/// injected so far, then `var` (the environment); one that cannot be +/// stays as written. A later file overrides an earlier one. +pub fn injected_env( + paths: &[PathBuf], + read: &dyn Fn(&Path) -> Option, + var: &dyn Fn(&str) -> Option, +) -> BTreeMap { + let mut vars: BTreeMap = BTreeMap::new(); + for path in paths { + let Some(text) = read(path) else { + continue; + }; + for (key, value) in parse_dotenv(&text) { + let value = expand_env(&value, &mut |name| { + Ok(vars.get(name).cloned().or_else(|| var(name))) + }) + .unwrap_or(value); + vars.insert(key, value); + } + } + vars +} + +/// The `KEY=value` pairs of the dotenv text `text`, in file order, as +/// yarn's dotenv parser reads them: an optional `export ` prefix, `=` or +/// `: ` between key and value, a value in single, double or back quotes +/// (which may span lines; a double-quoted one turns `\n` and `\r` into +/// line breaks), or else unquoted up to a `#` comment and trimmed. +pub fn parse_dotenv(text: &str) -> Vec<(String, String)> { + static LINE: std::sync::LazyLock = std::sync::LazyLock::new(|| { + regex::Regex::new( + r#"(?m)^\s*(?:export\s+)?([A-Za-z0-9_.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?$"#, + ) + .expect("the dotenv line pattern compiles") + }); + let text = text.replace("\r\n", "\n").replace('\r', "\n"); + LINE.captures_iter(&text) + .map(|caps| { + let mut value = caps.get(2).map_or("", |m| m.as_str()).trim().to_string(); + let quote = value.chars().next(); + if let Some(q) = quote.filter(|q| matches!(q, '\'' | '"' | '`')) { + if value.len() >= 2 && value.ends_with(q) { + value = value[1..value.len() - 1].to_string(); + } + if q == '"' { + value = value.replace("\\n", "\n").replace("\\r", "\r"); + } + } + (caps[1].to_string(), value) + }) + .collect() +} + /// The column-0 `key: value` line scan [`raw_setting`] falls back to: a /// quoted value ends at its closing quote, and a plain value ends before /// a whitespace-separated `#` comment (#370). @@ -217,6 +543,208 @@ mod tests { setting(rc, key, &env(vars)).map(|s| s.value) } + /// #1611: dotenv text as yarn's parser reads it. + #[test] + fn dotenv_reads_like_yarn() { + let text = "# comment\r\nA=1\nexport B = two words # note\n\ + C='single # kept'\nD=\"line\\nbreak\"\nE=`back`\nF=\nG: colon\n\ + H=\"multi\nline\"\nnot a pair\nI.J-K=x\n"; + let got: Vec<(String, String)> = parse_dotenv(text); + let want: Vec<(&str, &str)> = vec![ + ("A", "1"), + ("B", "two words"), + ("C", "single # kept"), + ("D", "line\nbreak"), + ("E", "back"), + ("F", ""), + ("G", "colon"), + ("H", "multi\nline"), + ("I.J-K", "x"), + ]; + let got: Vec<(&str, &str)> = got.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect(); + assert_eq!(got, want); + } + + /// #1611: `injectEnvironmentFiles` as a sequence, a flow list, a comma + /// string or emptied; another key's list is not it. + #[test] + fn list_settings_read_like_yarn() { + let key = "injectEnvironmentFiles"; + let list = |items: &[&str]| Some(items.iter().map(|s| s.to_string()).collect::>()); + assert_eq!( + raw_list_setting("injectEnvironmentFiles: [a, \"b?\"]\n", key), + list(&["a", "b?"]) + ); + assert_eq!( + raw_list_setting("x: [z]\ninjectEnvironmentFiles:\n - a\n - b\ny: 1\n", key), + list(&["a", "b"]) + ); + assert_eq!( + raw_list_setting("injectEnvironmentFiles: a, b\n", key), + list(&["a", " b"]) + ); + assert_eq!( + raw_list_setting("injectEnvironmentFiles: []\n", key), + list(&[]) + ); + assert_eq!( + raw_list_setting("x: [a]\nn:\n injectEnvironmentFiles: [b]\n", key), + None + ); + assert_eq!( + raw_list_setting( + "injectEnvironmentFiles: [a]\ninjectEnvironmentFiles: [b]\n", + key + ), + list(&["b"]) + ); + } + + /// #1611: the default list, an rc list against its folder, the + /// environment's list against the start folder, `?` and `${…}`. + #[test] + fn env_file_paths_follow_precedence() { + let start = Path::new("/p/app"); + let rc_dir = Path::new("/p"); + assert_eq!( + env_file_paths(None, &[], start), + vec![start.join(".env.yarn")] + ); + let rcs = [(rc_dir, "injectEnvironmentFiles: [\"${D:-env}/x\", y?]\n")]; + assert_eq!( + env_file_paths(None, &rcs, start), + vec![rc_dir.join("env/x"), rc_dir.join("y")] + ); + assert_eq!( + env_file_paths(Some("a, b?"), &rcs, start), + vec![start.join("a"), start.join(" b")] + ); + // Every rc file's list, lowest precedence (home, the farthest + // ancestor) first; a nearer `[]` clears nothing. + let (home, near) = (Path::new("/home/u"), Path::new("/p/app")); + let rcs = [ + (near, "injectEnvironmentFiles: []\n"), + (rc_dir, "injectEnvironmentFiles: [r]\n"), + (Path::new("/"), "nodeLinker: pnp\n"), + (home, "injectEnvironmentFiles: [h]\n"), + ]; + assert_eq!( + env_file_paths(None, &rcs, start), + vec![home.join("h"), rc_dir.join("r")] + ); + let rcs = [(near, "injectEnvironmentFiles: [n]\n"), rcs[1]]; + assert_eq!( + env_file_paths(None, &rcs, start), + vec![rc_dir.join("r"), near.join("n")] + ); + // Yarn 4.9.1 resolves a string's entries against the folder it + // runs in (a root `".env.local?"` reads `packages/app/.env.local`), + // expands `${…}` with no variable set (`${HOME:-.}` is `.` though + // HOME is exported), and keeps the spaces of a comma-split item. + let string = (rc_dir, "injectEnvironmentFiles: \".env.local?\"\n"); + assert_eq!( + env_file_paths(None, &[string], start), + vec![start.join(".env.local")] + ); + let set = (rc_dir, "injectEnvironmentFiles: [\"${HOME:-.}/x?\"]\n"); + assert_eq!( + env_file_paths(None, &[set], start), + vec![rc_dir.join("./x")] + ); + assert_eq!( + env_file_paths(Some("${HOME:-.}/x?"), &[], start), + vec![start.join("./x")] + ); + let spaced = (rc_dir, "injectEnvironmentFiles: \".env.a?, .env.b?\"\n"); + assert_eq!( + env_file_paths(None, &[spaced], start), + vec![start.join(".env.a"), start.join(" .env.b")] + ); + // `reset` keeps its own list and drops lower-precedence ones; + // `hardReset` drops its own too; a type change stops the walk. + let parent = (rc_dir, "injectEnvironmentFiles: [r]\n"); + let reset = ( + near, + "injectEnvironmentFiles: {onConflict: reset, value: [n]}\n", + ); + assert_eq!( + env_file_paths(None, &[reset, parent], start), + vec![near.join("n")] + ); + let reset = ( + near, + "injectEnvironmentFiles:\n onConflict: hardReset\n value:\n - n\n", + ); + assert_eq!( + env_file_paths(None, &[reset, parent], start), + Vec::::new() + ); + let string = (near, "injectEnvironmentFiles: n\n"); + assert_eq!( + env_file_paths(None, &[string, parent], start), + vec![near.join("n")] + ); + assert_eq!( + env_file_paths(None, &[parent, string], start), + vec![rc_dir.join("r")] + ); + // An entry yarn cannot expand is skipped. + let rcs = [(rc_dir, "injectEnvironmentFiles: [\"${UNSET}\", z]\n")]; + assert_eq!(env_file_paths(None, &rcs, start), vec![rc_dir.join("z")]); + } + + /// #1611: yarn expands a dotenv value's `${…}` against the variables + /// injected so far, then the environment; a later file wins. + #[test] + fn injected_values_expand_like_yarn() { + let files = [ + ("/a", "A=node-modules\nL=${A}\nM=${CI:-pnp}\nN=${UNSET}\n"), + ("/b", "B=${L}-x\n"), + ]; + let read = |p: &Path| { + files + .iter() + .find(|(f, _)| Path::new(f) == p) + .map(|(_, t)| t.to_string()) + }; + let paths = [ + PathBuf::from("/a"), + PathBuf::from("/missing"), + PathBuf::from("/b"), + ]; + let vars = injected_env(&paths, &read, &|n| (n == "CI").then(|| "pnpm".to_string())); + let get = |k: &str| vars.get(k).map(String::as_str); + assert_eq!(get("L"), Some("node-modules")); + assert_eq!(get("M"), Some("pnpm")); + assert_eq!(get("N"), Some("${UNSET}")); + assert_eq!(get("B"), Some("node-modules-x")); + } + + /// #1611: a list setting in a file yarn refuses is not read. + #[test] + fn list_settings_need_a_file_yarn_reads() { + let key = "injectEnvironmentFiles"; + assert_eq!( + raw_list_setting("- injectEnvironmentFiles\n- [a]\n", key), + None + ); + assert_eq!( + raw_list_setting( + "injectEnvironmentFiles: [a]\n---\ninjectEnvironmentFiles: []\n", + key + ), + None + ); + assert_eq!( + raw_list_setting("\u{feff}\u{feff}injectEnvironmentFiles: [a]\n", key), + None + ); + assert_eq!( + raw_list_setting("injectEnvironmentFiles:\n a: b\n", key), + None + ); + } + /// The #1100 shapes: yarn expands `${…}` in the value it uses. #[test] fn references_expand_like_yarn() { diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/npm.rs b/crates/socket-patch-core/src/patch/redirect/upstream/npm.rs index 8ebb95001..7686cd7a7 100644 --- a/crates/socket-patch-core/src/patch/redirect/upstream/npm.rs +++ b/crates/socket-patch-core/src/patch/redirect/upstream/npm.rs @@ -617,6 +617,9 @@ fn yarn_expand_env(value: &str, var: &dyn Fn(&str) -> Option) -> Result< struct BerryRegistrySettings { rcs: Vec, env_registry: Option, + /// The variables a value's `${NAME}` reads: this process's + /// environment under the project's injected dotenv files (#1611). + env: crate::crawlers::pkg_managers::YarnEnv, } impl BerryRegistrySettings { @@ -653,6 +656,7 @@ impl BerryRegistrySettings { BerryRegistrySettings { rcs, env_registry: std::env::var("YARN_NPM_REGISTRY_SERVER").ok(), + env: crate::crawlers::pkg_managers::YarnEnv::for_project(&root), } } @@ -661,7 +665,7 @@ impl BerryRegistrySettings { &self.rcs, self.env_registry.as_deref(), name, - &|key: &str| std::env::var(key).ok(), + &|key: &str| self.env.var(key), ) } } diff --git a/crates/socket-patch-core/src/patch/shared_store.rs b/crates/socket-patch-core/src/patch/shared_store.rs index a1159dfc6..9737885fa 100644 --- a/crates/socket-patch-core/src/patch/shared_store.rs +++ b/crates/socket-patch-core/src/patch/shared_store.rs @@ -349,11 +349,14 @@ fn in_yarn_pnpm_store(node_modules: &Path, real: &Path) -> bool { rc.ok().map(|rc| (dir, rc)) }) .collect(); + // Yarn expands `${…}` with the project's injected dotenv files applied + // (#1611). + let env = crate::crawlers::pkg_managers::YarnEnv::for_project(project); let nearest = |key: &str| { rcs.iter().find_map(|(dir, rc)| { // The value yarn uses (`${…}` expanded, #1100); one it cannot // expand admits nothing. - crate::formats::yarn::rc::process_setting(rc, key) + crate::formats::yarn::rc::setting(rc, key, &|name| env.var(name)) .map(|setting| (*dir, setting.value.unwrap_or_default())) }) }; diff --git a/crates/socket-patch-core/src/vendor/npm_flavor.rs b/crates/socket-patch-core/src/vendor/npm_flavor.rs index 7023f8c5d..c6130fb2a 100644 --- a/crates/socket-patch-core/src/vendor/npm_flavor.rs +++ b/crates/socket-patch-core/src/vendor/npm_flavor.rs @@ -1168,6 +1168,7 @@ mod tests { rc_filename: ".yarnrc.yml".into(), home: Some(home.path().to_path_buf()), vars: Some(Default::default()), + env_files: Default::default(), }; let (flavor, _) = detect_vendorable_npm_flavor_with(tmp.path(), || yarn_node_linker_in(tmp.path(), &env)) @@ -1183,6 +1184,7 @@ mod tests { rc_filename: ".yarnrc.ci.yml".into(), home: None, vars: Some(Default::default()), + env_files: Default::default(), }; let (flavor, _) = detect_vendorable_npm_flavor_with(tmp.path(), || yarn_node_linker_in(tmp.path(), &env)) @@ -1214,6 +1216,7 @@ mod tests { rc_filename: ".yarnrc.yml".into(), home: None, vars: Some([("SOCKET_PATCH_T_SECRET".to_string(), "secret".to_string())].into()), + env_files: Default::default(), }; let (code, message) = detect_vendorable_npm_flavor_with(tmp.path(), || { yarn_node_linker_in(tmp.path(), &env) diff --git a/docs/testing/yarn-berry-compatibility.md b/docs/testing/yarn-berry-compatibility.md index 22cd188a6..7e8eb3062 100644 --- a/docs/testing/yarn-berry-compatibility.md +++ b/docs/testing/yarn-berry-compatibility.md @@ -18,7 +18,9 @@ was rewired still runs the registry copy, so `vex` omits the package decided by the configured linker (`YARN_NODE_LINKER`, else the nearest rc file at or above the project that sets `nodeLinker`, else the home folder's rc file; the rc file is `.yarnrc.yml` unless `YARN_RC_FILENAME` renames it; unset means -berry's default, `pnp`), not by whether a `.pnp.*` loader happens to exist: +berry's default, `pnp`; a `${NAME}` in the value expands as yarn 4 expands it, +with the `injectEnvironmentFiles` files, `.env.yarn` by default, applied over +the environment), not by whether a `.pnp.*` loader happens to exist: `vendor` refuses a lock-only PnP checkout up front, and a stale `.pnp.js` left by a Yarn 2 migration to `node-modules` or `pnpm` is ignored. Yarn 1 PnP (`installConfig.pnp`, a classic `yarn.lock`) has no `nodeLinker`, so its