diff --git a/README.md b/README.md index 6f4e1ab..742922f 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ Tools used for publishing CESM input data. Notes: - Use `rimport --check` if you'd like to see the current status of a file, including whether it's available for download. - The `relink.py` script was previously used for step 3 above, but that functionality is now built into `rimport`. It's still there if you want to use it by itself. +- A relative filename passed to `rimport` directly (via `--file` or as a positional argument) is always resolved against your current directory — never against the inputdata root, and it doesn't matter whether you're running from inside or outside the inputdata tree. Pass an absolute path if you want to name a file without regard to your current directory. +- A relative entry in a `--list` file is always resolved against that list file's own directory — again never against the inputdata root, wherever the list file itself lives. Pass absolute entries in the list if you want them independent of the list file's location. +- Before staging anything, `rimport` validates every file it's about to process (all `--file`/`--list`/positional entries together). If any of them fail — missing, a directory, a broken symlink, outside the inputdata root, etc. — none of them are touched, and every failing path is reported at once so you can fix them all in one pass. This is a promise about rejected input, not about success: pre-flight passing doesn't guarantee the whole batch will finish, since a file can still fail later for a reason pre-flight can't see (e.g. a runtime/relink failure partway through). +- `--check` is gated by the same pre-flight validation as a real run: if any file in the batch fails validation, `rimport` reports the failures and exits without checking (or reporting on) any of the other files. This is deliberate, not a bug — fix the bad entries and re-run to see the rest. +- Exit codes: `0` means everything succeeded (or, under `--check`, everything checked cleanly); `2` means the run was rejected before touching anything (bad arguments, a missing/empty list file, or a pre-flight validation failure); `1` means pre-flight passed but something failed for real while actually being staged or relinked. ## Filenames and metadata: diff --git a/rimport b/rimport index 1024907..408287e 100755 --- a/rimport +++ b/rimport @@ -52,7 +52,11 @@ def build_parser() -> argparse.ArgumentParser: "-file", dest="file", metavar="filename", - help="Provide a file to import. Must be in the CESM inputdata directory.", + help=( + "Provide a file to import. Must be in the CESM inputdata directory. A relative name" + " is resolved against the current directory; there is no fallback to the" + " inputdata root." + ), ) parser.add_argument( @@ -62,14 +66,19 @@ def build_parser() -> argparse.ArgumentParser: metavar="filelist", help=( "Provide a file that contains a list of filenames to import. All filenames in the list" - " must be in the CESM inputdata directory." + " must be in the CESM inputdata directory. A relative entry is resolved against the" + " list file's own directory, wherever that directory is." ), ) parser.add_argument( "items_to_process", nargs="*", - help="One or more files to process. (Optional; can use --file instead to process just one.)" + help=( + "One or more files to process. (Optional; can use --file instead to process just one.)" + " A relative name is resolved against the current directory; there is no fallback to" + " the inputdata root." + ), ) # Add inputdata_root option flags @@ -80,7 +89,11 @@ def build_parser() -> argparse.ArgumentParser: "-check", "-c", action="store_true", - help="Check whether file(s) is/are already published.", + help=( + "Check whether file(s) is/are already published, without staging anything. A bad" + " path anywhere in the batch aborts before any file is checked, reporting all bad" + " paths at once." + ), ) # Add verbosity options @@ -119,13 +132,24 @@ def normalize_paths(root: Path, relnames: Iterable[str]) -> List[Path]: """Convert relative or absolute path names to normalized absolute Paths. For each name in relnames: - - If the name is relative, it is assumed to be relative to `root` and made absolute + - If the name is relative, it is joined onto `root` and made absolute. All paths are then normalized to their absolute form, replacing . and .. as needed. + The `root / name` branch above is unreachable from `main`: `get_files_to_process` now + anchors every relative name before returning it -- CLI-style names (`--file`, positional) + against cwd, `--list` entries against the list file's own directory -- so nothing relative + ever reaches this function on `main`'s call path, and `root` is effectively unused there. + This is dead code, kept deliberately rather than removed; do not read this function in + isolation and conclude that root-relative resolution is still reachable anywhere in + `rimport`. The branch is still real code, though: it is exercised directly by this + function's own unit tests in test_normalize_paths.py, which call normalize_paths() with + relative names and a `root` of their choosing. + Note that symlinks are NOT resolved. Args: - root: Base directory under which relative paths are assumed to be. + root: Base directory under which relative paths are assumed to be. Only matters for + the unreachable-from-`main` branch described above. relnames: Iterable of path names (relative or absolute) to normalize. Returns: @@ -153,6 +177,67 @@ def check_relink_worked(src: Path, dst: Path) -> None: raise RuntimeError("Error relinking during rimport") +def validate_source_path( + src: Path, inputdata_root: Path, staging_root: Path +) -> Exception | None: + """Run stage_data's read-only guardrails against `src` without raising or logging. + + This is the pre-flight half of `stage_data`'s checks: broken symlink, live symlink whose + target is outside staging, missing file, directory, outside the inputdata root, and already + under the staging directory. It performs no I/O beyond stat-ing `src` and its resolved + target — in particular it never makes a network call — so it is cheap to run over an entire + batch before anything is staged. + + Critically, a *live* symlink whose target resolves under `staging_root` is NOT a failure: + that is the normal state of a file that has already been published and linked by a previous + run, and is treated here exactly like a plain, stageable file (returns `None`). Re-running + rimport over an already-published tree must not report every published file as bad. + + `stage_data` calls this first and raises whatever it returns; `main`'s pre-flight gate calls + it over every resolved path before staging anything, so a bad path anywhere in the batch is + reported without partially publishing the rest. + + Args: + src: Source file path to validate. + inputdata_root: Root directory of the inputdata tree. + staging_root: Root directory where files will be staged. + + Returns: + The exception (unraised) describing why `src` cannot be staged, or `None` if `src` is + fine to proceed with — including the already-published-and-linked symlink case. + """ + if src.is_symlink(): + if not os.path.exists(src.resolve()): + return RuntimeError(f"Source is a broken symlink: {src}") + if not src.resolve().is_relative_to(staging_root.resolve()): + return RuntimeError( + f"Source is a symlink, but target ({src.resolve()}) is outside staging directory " + f"({staging_root})" + ) + # Live symlink already resolving under staging_root: already published and linked. + # This is a legitimate no-op, not a validation failure. + return None + + if not src.exists(): + return FileNotFoundError(f"source not found: {src}") + + if src.is_dir(): + return RuntimeError(f"source is a directory, not a file: {src}") + + try: + src.resolve().relative_to(inputdata_root.resolve()) + except ValueError: + if src.resolve().is_relative_to(staging_root.resolve()): + return RuntimeError( + f"Source file '{src.name}' is already under staging directory '{staging_root}'." + ) + return RuntimeError( + f"source not under inputdata root: {src} not in {inputdata_root}" + ) + + return None + + def stage_data( src: Path, inputdata_root: Path, staging_root: Path, check: bool = False ) -> None: @@ -172,41 +257,43 @@ def stage_data( RuntimeError: If `src` is a live symlink pointing outside staging, or if `src` is outside the inputdata root, or if `src` is already under staging directory. RuntimeError: If `src` is a broken symlink. + RuntimeError: If `src` is a directory. This check runs only when `src` is not itself + a symlink — see the symlink guardrails above, which return early (without + raising) for a symlink whose target is a directory under `staging_root`. + A symlink whose directory target is outside `staging_root` instead raises + via the first entry above. RuntimeError: If it failed to replace `src` with a symlink to the staged file. FileNotFoundError: If `src` does not exist. Guardrails: - * Raise if `src` is a *live* symlink to a file outside staging root ("outside staging"). + * Raise if `src` is a *live* symlink to a target outside staging root ("outside staging"). * Raise if `src` is a broken symlink or is outside the inputdata root. + * Raise if `src` is a directory and not itself a symlink, so a non-symlink directory + source can never reach the replace-with-symlink path (which assumes a regular file + and mangles a directory). A *symlink* whose target is a directory is NOT covered by + this guardrail: it is handled by the live-symlink guardrails above instead, which for + a target under `staging_root` log "already published and linked" and return without + raising, and for a target outside `staging_root` raise (see the first guardrail above). + + These read-only guardrails are delegated to `validate_source_path`, which returns the + exception to raise (or None); this function raises whatever comes back. That keeps this + function and `main`'s pre-flight gate in permanent agreement about what is stageable, + since both call the same check. """ + error = validate_source_path(src, inputdata_root, staging_root) + if error is not None: + raise error + if src.is_symlink(): - if not os.path.exists(src.resolve()): - raise RuntimeError(f"Source is a broken symlink: {src}") - if not src.resolve().is_relative_to(staging_root.resolve()): - raise RuntimeError( - f"Source is a symlink, but target ({src.resolve()}) is outside staging directory " - f"({staging_root})" - ) + # validate_source_path only lets a symlink through here when it is live and its target + # resolves under staging_root: the "already published and linked" case. logger.info("%sFile is already published and linked.", INDENT) print_can_file_be_downloaded( can_file_be_downloaded(src.resolve(), staging_root) ) return - if not src.exists(): - raise FileNotFoundError(f"source not found: {src}") - - try: - rel = src.resolve().relative_to(inputdata_root.resolve()) - except ValueError as exc: - if src.resolve().is_relative_to(staging_root.resolve()): - raise RuntimeError( - f"Source file '{src.name}' is already under staging directory '{staging_root}'." - ) from exc - raise RuntimeError( - f"source not under inputdata root: {src} not in {inputdata_root}" - ) from exc - + rel = src.resolve().relative_to(inputdata_root.resolve()) dst = staging_root / rel if dst.exists(): @@ -336,6 +423,18 @@ def get_files_to_process(file: str, filelist: str, items_to_process: list): Uses --file and/or --filelist arguments, as well as positional items_to_process if given. + One rule, no modes, no root-relative fallback: non-absolute `file` and + `items_to_process` entries are CLI args and always anchor (eagerly, absolute) to cwd. + Non-absolute `--list` entries always anchor (eagerly, absolute) against the list + file's own directory. Absolute paths are returned unchanged everywhere. + + If `Path.cwd()` itself raises (e.g. the working directory was deleted out from under + the process) and at least one `file`/`items_to_process` entry is relative, there is + nothing to anchor it against: this is fatal, reporting every offending relative name + in one error message and returning `(None, 2)`. Absolute `file`/`items_to_process` + entries, and all `--list` entries (which never anchor to cwd), are unaffected by an + undeterminable cwd. + Args: file (str): Single file to process. filelist (str): File containing list of files to process. @@ -345,10 +444,30 @@ def get_files_to_process(file: str, filelist: str, items_to_process: list): list: List of files to process int: Result code """ - if file is not None: - files_to_process = [file] - else: - files_to_process = [] + cli_args = ([file] if file is not None else []) + list(items_to_process or []) + relative_cli_args = [name for name in cli_args if not Path(name).is_absolute()] + + cwd = None + if relative_cli_args: + try: + cwd = Path.cwd().resolve() + except OSError: + # cwd may have been deleted out from under the process (FileNotFoundError, itself + # an OSError). There is no root fallback to anchor a relative name against instead, + # so this is fatal: report every offending name in one error message. + logger.error( + "rimport: could not determine the current working directory (it may have " + "been deleted); cannot resolve relative name(s): %s", + ", ".join(relative_cli_args), + ) + return None, 2 + + def _anchor_cli(name): + if Path(name).is_absolute(): + return name + return str(cwd / name) + + files_to_process = [_anchor_cli(file)] if file is not None else [] if filelist is not None: list_path = Path(filelist).expanduser().resolve() @@ -359,10 +478,16 @@ def get_files_to_process(file: str, filelist: str, items_to_process: list): if not files_in_list: logger.error("rimport: no filenames found in list: %s", list_path) return None, 2 - files_to_process.extend(files_in_list) + + list_base = list_path.parent + for entry in files_in_list: + if Path(entry).is_absolute(): + files_to_process.append(entry) + else: + files_to_process.append(str(list_base / entry)) if items_to_process: - files_to_process.extend(items_to_process) + files_to_process.extend(_anchor_cli(item) for item in items_to_process) if not files_to_process: logger.error("rimport: At least one of --file or --filelist is required") @@ -389,9 +514,26 @@ def main(argv: List[str] | None = None) -> int: RIMPORT_STAGING: Override the default staging root directory. Exit Codes: - 0: All files staged successfully. - 1: One or more files failed to stage or relink (errors printed to stderr). - 2: Fatal error (missing inputdata directory, missing file list, etc.). + 0: All files staged successfully (or, under --check, checked without error). + 1: Pre-flight validation passed, but one or more files failed while actually being + staged or relinked -- a genuine runtime failure, not a rejected input (errors + printed to stderr for each). + 2: The run was rejected before any file was staged or checked. Causes: + * argparse rejected the command line itself -- e.g. an unrecognized option, + or an --inputdata-root that does not exist (argparse validates it via + type=, so this fires before any of the code below runs). + * The inputdata root directory does not exist. + * No --file, --list, or positional argument was given. + * The --list file was not found, or contained no filenames. + * A relative --file or positional name could not be anchored, because the + current working directory could not be determined. + * User-switching to STAGE_OWNER failed: target user not found, or no TTY + available for the sudo/2FA prompt. + * Most commonly: pre-flight validation rejected one or more of the resolved + paths (missing file, directory, broken symlink, source outside the + inputdata root, etc.). Pre-flight checks every resolved path before + staging anything and reports every failure at once, so a bad path never + leaves the batch half-published. This gate applies to --check runs too. """ parser = build_parser() args = parser.parse_args(argv) @@ -412,13 +554,34 @@ def main(argv: List[str] | None = None) -> int: return 2 # Determine the list of relative filenames to handle - files_to_process, status = get_files_to_process(args.file, args.filelist, args.items_to_process) + files_to_process, status = get_files_to_process( + args.file, args.filelist, args.items_to_process + ) if status: return status # Resolve to full paths (keep accepting absolute names too) paths = normalize_paths(root, files_to_process) staging_root = get_staging_root() + + # Pre-flight: validate every path before staging anything, so a batch containing a bad + # path is reported (all bad paths at once) instead of half-completing. Runs for --check + # too: a bad entry aborts the whole batch rather than being reported per-file. + failures = [] + for p in paths: + error = validate_source_path(p, root, staging_root) + if error is not None: + failures.append((p, error)) + if failures: + logger.error( + "rimport: %d of %d file(s) failed pre-flight validation; nothing was published:", + len(failures), + len(paths), + ) + for p, error in failures: + logger.error("%srimport: '%s': %s", INDENT, p, error) + return 2 + # Execute the new action per file errors = 0 for p in paths: diff --git a/tests/conftest.py b/tests/conftest.py index 0a7e232..eef610e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,3 +31,28 @@ def fixture_temp_dirs(): # Cleanup shutil.rmtree(source_dir, ignore_errors=True) shutil.rmtree(target_dir, ignore_errors=True) + + +@pytest.fixture(scope="function", name="nested_mock_dirs") +def fixture_nested_mock_dirs(tmp_path): + """Create a nested source/target directory layout for testing relative-path + resolution.""" + source_dir = tmp_path / "source" + target_dir = tmp_path / "target" + source_sub_dir = source_dir / "sub" + target_sub_dir = target_dir / "sub" + source_sub_dir.mkdir(parents=True) + target_sub_dir.mkdir(parents=True) + + # Create a test file + source_file = source_sub_dir / "test_file.txt" + target_file = target_sub_dir / "test_file.txt" + source_file.write_text("source content") + target_file.write_text("target content") + + yield source_dir, target_dir, source_sub_dir, source_file, target_file + + # No explicit cleanup: everything above was created under tmp_path, which + # pytest already removes automatically. Unlike fixture_temp_dirs (which + # uses tempfile.mkdtemp, a directory pytest does not manage), an explicit + # shutil.rmtree here would be redundant. diff --git a/tests/relink/test_cmdline.py b/tests/relink/test_cmdline.py index 46a03c8..a6bca83 100644 --- a/tests/relink/test_cmdline.py +++ b/tests/relink/test_cmdline.py @@ -136,6 +136,132 @@ def test_command_line_execution_given_file(mock_dirs): assert f"{INDENT}Created symbolic link:" in result.stdout +def _run_relink_relative_positional(nested_mock_dirs, positional_arg): + """Create a same-named decoy at the inputdata root (outside "sub"), then + run relink.py with `positional_arg` as the sole path argument and cwd + set to the inputdata subdirectory. + + Shared setup and invocation for + test_command_line_relative_file_from_inputdata_subdir and + test_command_line_relative_dir_dot_from_inputdata_subdir. Those two + tests differ ONLY in `positional_arg` (a bare filename vs "."), and that + single difference changes what regression each one discriminates and + which of its own assertions does the discriminating -- see each test's + docstring. This helper deliberately makes no assertion about the + outcome of the run beyond confirming the decoy setup itself: the payoff + assertions that make each test meaningful differ between the two + callers and stay in the tests, not here. + + Returns (result, decoy_file) for the caller to assert against. + """ + source_dir, target_dir, source_sub_dir, source_file, target_file = ( + nested_mock_dirs + ) + + # Decoy file directly under the inputdata root (outside "sub"), same + # name as the intended file but different content, with a matching + # target copy (also different content). + decoy_file = source_dir / source_file.name + decoy_target = target_dir / target_file.name + decoy_file.write_text("decoy content") + decoy_target.write_text("decoy target content") + assert decoy_file.read_text() != source_file.read_text() + assert decoy_target.read_text() != target_file.read_text() + + # Get the path to relink.py + relink_script = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "relink.py", + ) + + # Build the command + command = [ + sys.executable, + relink_script, + positional_arg, + "--target-root", + str(target_dir), + "--inputdata-root", + str(source_dir), + ] + + # Execute the command with cwd set to the inputdata subdirectory + result = subprocess.run( + command, cwd=str(source_sub_dir), capture_output=True, text=True, check=False + ) + + return result, decoy_file + + +def test_command_line_relative_file_from_inputdata_subdir(nested_mock_dirs): + """Test that a bare relative filename is resolved against the cwd (an + inputdata subdirectory), not against the inputdata root. + + A same-named decoy file sits directly under the inputdata root, outside + "sub", with different content than the intended file, plus a matching + target copy (also with different content) so it *would* be relinked if a + root-relative regression resolved "test_file.txt" against inputdata_root + instead of cwd. Because this is a single-file argument rather than a + directory to recurse into, such a regression would process the decoy + INSTEAD OF the subdir file, not in addition to it. The return code and + decoy_file.is_file() pass either way (the latter because is_file() + follows a symlink to a real file); what actually discriminates is that + the decoy stays a plain file with its original content (not relinked), + and that the intended subdir file is the one converted to a symlink -- + pointing at the subdir's target copy, not the root decoy's. + """ + *_, source_file, target_file = nested_mock_dirs + + result, decoy_file = _run_relink_relative_positional( + nested_mock_dirs, source_file.name + ) + + # Verify the command executed successfully + assert result.returncode == 0, f"Command failed with stderr: {result.stderr}" + + # Verify the intended subdir file was converted to a symlink pointing at + # the subdir's target copy (not the root decoy's) + assert source_file.is_symlink() + assert os.readlink(str(source_file)) == str(target_file) + + # Verify the decoy at the inputdata root was NOT reached/relinked + assert decoy_file.is_file() + assert not decoy_file.is_symlink() + assert decoy_file.read_text() == "decoy content" + + +def test_command_line_relative_dir_dot_from_inputdata_subdir(nested_mock_dirs): + """Test that '.' is resolved against the cwd (an inputdata subdirectory), + not against the inputdata root. + + A same-named decoy file sits directly under the inputdata root, outside + "sub", with different content than the intended file, plus a matching + target copy (also with different content) so it *would* be relinkable if + reached. This test's discriminator is LOCATION, not the name collision: + because relink's search is recursive, resolving '.' against cwd + (source_dir/sub) never reaches the decoy, while a root-relative + regression -- resolving '.' against inputdata_root (source_dir) instead + -- would recurse into the decoy too. Only the decoy assertion below + actually discriminates between those two resolutions; the symlink-target + subdirectory (source_file) is reached by recursion either way. + """ + *_, source_file, target_file = nested_mock_dirs + + result, decoy_file = _run_relink_relative_positional(nested_mock_dirs, ".") + + # Verify the command executed successfully + assert result.returncode == 0, f"Command failed with stderr: {result.stderr}" + + # Verify the file was converted to a symlink pointing at the target copy + assert source_file.is_symlink() + assert os.readlink(str(source_file)) == str(target_file) + + # Verify the decoy outside "sub" was NOT reached/relinked + assert decoy_file.is_file() + assert not decoy_file.is_symlink() + assert decoy_file.read_text() == "decoy content" + + def test_command_line_multiple_source_dirs(temp_dirs): """Test executing relink.py with multiple source directories.""" inputdata_dir, target_dir = temp_dirs diff --git a/tests/rimport/test_cmdline.py b/tests/rimport/test_cmdline.py index be05c5d..d08d89e 100644 --- a/tests/rimport/test_cmdline.py +++ b/tests/rimport/test_cmdline.py @@ -61,7 +61,7 @@ def test_file_option_stages_single_file( sys.executable, rimport_script, "-file", - "test.nc", + str(test_file), "-inputdata", str(inputdata_root), ] @@ -100,9 +100,9 @@ def test_list_option_stages_multiple_files( file1.write_text("data1") file2.write_text("data2") - # Create filelist + # Create filelist (outside the tree; absolute entries) filelist = tmp_path / "filelist.txt" - filelist.write_text("file1.nc\nfile2.nc\n") + filelist.write_text(f"{file1}\n{file2}\n") # Run rimport with -list option command = [ @@ -137,6 +137,128 @@ def test_list_option_stages_multiple_files( assert file2.is_symlink() assert file2.resolve() == (staging_root / "file2.nc") + def test_list_inside_tree_relative_entries( + self, rimport_script, test_env, rimport_env + ): + """Test that a list file inside the inputdata tree anchors relative entries to the + list file's own directory, not the inputdata root.""" + inputdata_root = test_env["inputdata_root"] + staging_root = test_env["staging_root"] + + # Create nested file and a list file alongside it inside the tree + nested_file = inputdata_root / "lnd" / "clm2" / "file1.nc" + nested_file.parent.mkdir(parents=True) + nested_file.write_text("nested data") + + filelist = inputdata_root / "lnd" / "filelist.txt" + filelist.write_text(f"clm2/{nested_file.name}\n") + + # Decoy at the root-anchored location: a root-relative regression would resolve + # here instead, giving a wrong-file failure rather than a merely-missing-file one. + decoy_file = inputdata_root / "clm2" / nested_file.name + decoy_file.parent.mkdir(parents=True) + decoy_file.write_text("decoy data") + assert decoy_file.read_text() != nested_file.read_text() + + # Run rimport with -list option + command = [ + sys.executable, + rimport_script, + "-list", + str(filelist), + "-inputdata", + str(inputdata_root), + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + ) + + # Verify success + assert result.returncode == 0, f"Command failed: {result.stderr}" + + # Verify the file was staged, anchored to the list dir's own subtree + staged_file = staging_root / "lnd" / "clm2" / "file1.nc" + assert staged_file.exists() + assert staged_file.read_text() == "nested data" + + # Verify file was relinked + assert nested_file.is_symlink() + assert nested_file.resolve() == staged_file + + # Verify the decoy was left untouched, and nothing staged at the root-anchored path + assert not decoy_file.is_symlink() + assert decoy_file.read_text() == "decoy data" + assert not (staging_root / "clm2" / nested_file.name).exists() + + def test_list_inside_tree_relative_entries_anchor_to_list_dir_not_cwd( + self, rimport_script, test_env, rimport_env + ): + """Test that a list file's relative entries anchor to the list file's own directory, + not the cwd, even when rimport is run with its cwd inside the tree at a DIFFERENT + location. test_list_inside_tree_relative_entries above passes no cwd= to + subprocess.run, so pytest's own (outside-the-tree) cwd applies and cwd-anchoring and + list-dir-anchoring agree; this test sets cwd= explicitly so the two schemes can be + told apart.""" + inputdata_root = test_env["inputdata_root"] + staging_root = test_env["staging_root"] + + # Real file, alongside the list file inside "lnd" + nested_file = inputdata_root / "lnd" / "clm2" / "file1.nc" + nested_file.parent.mkdir(parents=True) + nested_file.write_text("real data") + + filelist = inputdata_root / "lnd" / "filelist.txt" + filelist.write_text(f"clm2/{nested_file.name}\n") + + # Decoy at the cwd-anchored location: a cwd-anchoring regression would resolve here + # instead, giving a wrong-file failure rather than a merely-missing-file one. + decoy_file = inputdata_root / "atm" / "clm2" / nested_file.name + decoy_file.parent.mkdir(parents=True) + decoy_file.write_text("decoy data") + assert decoy_file.read_text() != nested_file.read_text() + + # Run rimport with -list option, cwd inside the tree but at a DIFFERENT location + # than the list file + command = [ + sys.executable, + rimport_script, + "-list", + str(filelist), + "-inputdata", + str(inputdata_root), + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + cwd=inputdata_root / "atm", + ) + + # Verify success + assert result.returncode == 0, f"Command failed: {result.stderr}" + + # Verify the real file (not the decoy) was staged, anchored to the list dir's subtree + staged_file = staging_root / "lnd" / "clm2" / nested_file.name + assert staged_file.exists() + assert staged_file.read_text() == "real data" + + # Verify file was relinked + assert nested_file.is_symlink() + assert nested_file.resolve() == staged_file + + # Verify the decoy was left untouched, and nothing staged at the cwd-anchored path + assert not decoy_file.is_symlink() + assert decoy_file.read_text() == "decoy data" + assert not (staging_root / "atm" / "clm2" / nested_file.name).exists() + def test_preserves_directory_structure(self, rimport_script, test_env, rimport_env): """Test that directory structure is preserved in staging.""" inputdata_root = test_env["inputdata_root"] @@ -152,7 +274,7 @@ def test_preserves_directory_structure(self, rimport_script, test_env, rimport_e sys.executable, rimport_script, "-file", - "dir1/dir2/file.nc", + str(nested_file), "-inputdata", str(inputdata_root), ] @@ -186,7 +308,7 @@ def test_error_for_nonexistent_file(self, rimport_script, test_env, rimport_env) sys.executable, rimport_script, "-file", - "nonexistent.nc", + str(inputdata_root / "nonexistent.nc"), "-inputdata", str(inputdata_root), ] @@ -199,9 +321,12 @@ def test_error_for_nonexistent_file(self, rimport_script, test_env, rimport_env) env=rimport_env, ) - # Verify error + # Verify error. Assert the actual reason, not the substring "error": pre-flight + # reports "N of M file(s) failed pre-flight validation", which contains no such + # word, so a bare "error" check is satisfied only by tmp_path echoing this test's + # own name back in the offending path. assert result.returncode != 0 - assert "error" in result.stderr.lower() + assert "source not found" in result.stderr def test_error_for_nonexistent_list_file( self, rimport_script, test_env, rimport_env @@ -293,9 +418,9 @@ def test_list_with_comments_and_blanks(self, rimport_script, test_env, rimport_e file1.write_text("data1") file2.write_text("data2") - # Create filelist with comments and blanks + # Create filelist with comments and blanks (outside the tree; absolute entries) filelist = tmp_path / "filelist.txt" - filelist.write_text("# Comment\nfile1.nc\n\n# Another comment\nfile2.nc\n") + filelist.write_text(f"# Comment\n{file1}\n\n# Another comment\n{file2}\n") # Run rimport command = [ @@ -350,7 +475,7 @@ def test_prints_and_exits_for_already_published_linked_file( sys.executable, rimport_script, "-file", - "link.nc", + str(src), "-inputdata", str(inputdata_root), ] @@ -393,7 +518,7 @@ def test_error_broken_symlink(self, rimport_script, test_env, rimport_env): sys.executable, rimport_script, "-file", - "link.nc", + str(src), "-inputdata", str(inputdata_root), ] @@ -435,7 +560,7 @@ def test_error_symlink_pointing_outside_staging( sys.executable, rimport_script, "-file", - "link.nc", + str(src), "-inputdata", str(inputdata_root), ] @@ -473,7 +598,7 @@ def test_check_doesnt_copy_unpublished(self, rimport_script, test_env, rimport_e sys.executable, rimport_script, "-file", - file_basename, + str(test_file), "-inputdata", str(inputdata_root), "--check", @@ -506,6 +631,254 @@ def test_check_doesnt_copy_unpublished(self, rimport_script, test_env, rimport_e assert "Created symbolic link".lower() not in result.stdout.lower() assert "Error creating symlink".lower() not in result.stdout.lower() + def test_relative_file_from_inputdata_subdir_stages_that_file( + self, rimport_script, test_env, rimport_env + ): + """Test that a relative positional filename anchors to cwd when rimport is run from + inside an inputdata subdirectory, staging the file that is actually there rather than + a same-named decoy file at the inputdata root.""" + inputdata_root = test_env["inputdata_root"] + staging_root = test_env["staging_root"] + + subdir = inputdata_root / "lnd" / "clm2" + subdir.mkdir(parents=True) + + file_basename = "test.nc" + subdir_file = subdir / file_basename + subdir_file.write_text("subdir data") + + decoy_file = inputdata_root / file_basename + decoy_file.write_text("decoy data") + assert decoy_file.read_text() != subdir_file.read_text() + + # Run rimport with a relative positional filename, from inside the subdir + command = [ + sys.executable, + rimport_script, + file_basename, + "-inputdata", + str(inputdata_root), + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + cwd=subdir, + ) + + # Verify success + assert result.returncode == 0, f"Command failed: {result.stderr}" + + # Verify the subdir file (not the decoy) was staged + staged_file = staging_root / "lnd" / "clm2" / file_basename + assert staged_file.exists() + assert staged_file.read_text() == "subdir data" + + # Verify the subdir file was relinked + assert subdir_file.is_symlink() + assert subdir_file.resolve() == staged_file + + # Verify the decoy at the inputdata root was left untouched + assert not decoy_file.is_symlink() + assert decoy_file.read_text() == "decoy data" + + # Verify nothing was staged at the root-anchored path + assert not (staging_root / file_basename).exists() + + def test_relative_file_from_subdir_missing_errors_no_root_fallback( + self, rimport_script, test_env, rimport_env + ): + """Test that a relative positional filename run from an inputdata subdirectory errors + when the file isn't there, rather than falling back to a same-named file at the root.""" + inputdata_root = test_env["inputdata_root"] + staging_root = test_env["staging_root"] + + subdir = inputdata_root / "lnd" / "clm2" + subdir.mkdir(parents=True) + + file_basename = "test.nc" + root_file = inputdata_root / file_basename + root_file.write_text("root data") + + # Run rimport with a relative positional filename, from inside the subdir + command = [ + sys.executable, + rimport_script, + file_basename, + "-inputdata", + str(inputdata_root), + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + cwd=subdir, + ) + + # Verify failure + assert result.returncode != 0, f"Command unexpectedly passed: {result.stdout}" + assert "source not found" in result.stderr + + # Verify the root file was left untouched + assert not root_file.is_symlink() + assert root_file.read_text() == "root data" + + # Verify nothing was staged + assert not any(staging_root.iterdir()) + + def test_relative_file_from_outside_tree_errors_no_root_fallback( + self, rimport_script, test_env, rimport_env + ): + """Test that a relative positional filename run from OUTSIDE the inputdata tree anchors + to cwd and errors, rather than falling back to a same-named file at the inputdata root. + + An outside-the-tree cwd is essential here: with cwd inside the tree, root-anchored and + cwd-anchored resolution agree, so the sibling tests above would pass either way. Only + placing cwd outside the tree discriminates root-anchored resolution (which would + silently stage the root file, rc 0) from cwd-anchored resolution (error, stage + nothing). The decoy is what makes it discriminating: without a file at the + root-anchored path there would be nothing for a regression to wrongly publish. + """ + inputdata_root = test_env["inputdata_root"] + staging_root = test_env["staging_root"] + tmp_path = test_env["tmp_path"] + + outside = tmp_path / "outside" + outside.mkdir() + + file_basename = "test.nc" + decoy_file = inputdata_root / file_basename + decoy_file.write_text("decoy data") + + # Run rimport with a relative positional filename, from outside the tree + command = [ + sys.executable, + rimport_script, + file_basename, + "-inputdata", + str(inputdata_root), + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + cwd=outside, + ) + + # Verify failure. Deliberately not pinning the exact return code, since it may change + # in the future for any reason. + assert result.returncode != 0, f"Command unexpectedly passed: {result.stdout}" + + # Verify the decoy at the inputdata root was NOT published + assert not decoy_file.is_symlink() + assert decoy_file.read_text() == "decoy data" + + # Verify nothing was staged + assert not any(staging_root.iterdir()) + + def test_dotdot_escape_from_subdir_errors( + self, rimport_script, test_env, rimport_env + ): + """Test that a '..'-escaping relative filename, anchored lexically to cwd, is rejected + by stage_data's existing outside-the-root guardrail.""" + inputdata_root = test_env["inputdata_root"] + tmp_path = test_env["tmp_path"] + + subdir = inputdata_root / "lnd" + subdir.mkdir(parents=True) + + # File must exist, or the "source not found" check fires before the guardrail and the + # "not under inputdata root" message never appears. + outside_file = tmp_path / "outside.nc" + outside_file.write_text("outside data") + + # Run rimport with an escaping relative positional filename, from inside the subdir + command = [ + sys.executable, + rimport_script, + "../../outside.nc", + "-inputdata", + str(inputdata_root), + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + cwd=subdir, + ) + + # Verify failure + assert result.returncode != 0, f"Command unexpectedly passed: {result.stdout}" + assert "not under inputdata root" in result.stderr + + # Verify the outside file was left untouched + assert not outside_file.is_symlink() + assert outside_file.read_text() == "outside data" + + def test_dotdot_escape_from_list_inside_tree_errors( + self, rimport_script, test_env, rimport_env + ): + """Test that a '..'-escaping relative entry in a list file INSIDE the tree is rejected + by validate_source_path's outside-the-root guardrail, via main()'s pre-flight gate. + This is the list-side twin of test_dotdot_escape_from_subdir_errors above.""" + inputdata_root = test_env["inputdata_root"] + staging_root = test_env["staging_root"] + tmp_path = test_env["tmp_path"] + + list_dir = inputdata_root / "lnd" + list_dir.mkdir(parents=True) + + # File must exist, or the "source not found" check fires before the guardrail and the + # "not under inputdata root" message never appears. + outside_file = tmp_path / "outside.nc" + outside_file.write_text("outside data") + + # List file INSIDE the tree, with an entry that escapes the tree + filelist = list_dir / "filelist.txt" + filelist.write_text("../../outside.nc\n") + + # Run rimport with -list option + command = [ + sys.executable, + rimport_script, + "-list", + str(filelist), + "-inputdata", + str(inputdata_root), + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + ) + + # Verify failure. rc 2 (not 1): pre-flight validation catches this before anything is + # staged, rather than the per-file loop catching it after some files may have run. + assert result.returncode == 2, f"Command unexpectedly passed: {result.stdout}" + assert "not under inputdata root" in result.stderr + + # Verify nothing was staged + assert not any(staging_root.rglob("*")) + + # Verify the outside file was left untouched + assert not outside_file.is_symlink() + assert outside_file.read_text() == "outside data" + def test_check_doesnt_relink_published(self, rimport_script, test_env, rimport_env): """Test that published file is not relinked if check is True.""" inputdata_root = test_env["inputdata_root"] @@ -526,7 +899,7 @@ def test_check_doesnt_relink_published(self, rimport_script, test_env, rimport_e sys.executable, rimport_script, "-file", - file_basename, + str(test_file), "-inputdata", str(inputdata_root), "--check", @@ -554,3 +927,269 @@ def test_check_doesnt_relink_published(self, rimport_script, test_env, rimport_e assert "Deleted original file".lower() not in result.stdout.lower() assert "Created symbolic link".lower() not in result.stdout.lower() assert "Error creating symlink".lower() not in result.stdout.lower() + + def test_directory_argument_from_subdir_errors_and_leaves_tree_intact( + self, rimport_script, test_env, rimport_env + ): + """Test that pointing rimport at a directory (e.g. via the cwd-anchored positional from + inside an inputdata subdir) errors cleanly instead of falling into the destructive + replace-with-symlink path, which would rename the directory to '.tmp', symlink it + away, and then fail to roll back.""" + inputdata_root = test_env["inputdata_root"] + staging_root = test_env["staging_root"] + + subdir = inputdata_root / "lnd" / "clm2" + subdir.mkdir(parents=True) + inner_file = subdir / "data.nc" + inner_file.write_text("clm2 data") + + # The matching staging mirror must ALSO exist as a directory, or dst.exists() is False + # and stage_data takes the harmless "not already published" branch instead of the + # destructive one — this is the fixture detail that makes the bug actually bite. + staging_mirror = staging_root / "lnd" / "clm2" + staging_mirror.mkdir(parents=True) + + # Run rimport with a relative positional directory name, from inside the parent dir + command = [ + sys.executable, + rimport_script, + "clm2", + "-inputdata", + str(inputdata_root), + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + cwd=subdir.parent, + ) + + # Verify failure + assert result.returncode != 0, f"Command unexpectedly passed: {result.stdout}" + assert "directory" in result.stderr + assert "not a file" in result.stderr + + # Verify the tree is intact: clm2 is still a real directory, not a symlink; no + # '.tmp' path was left anywhere under inputdata_root; and its contents are + # untouched. + assert subdir.is_dir() and not subdir.is_symlink(), ( + f"clm2 should still be a plain, non-symlink directory after the error; " + f"is_dir={subdir.is_dir()} is_symlink={subdir.is_symlink()}" + ) + tmp_paths = list(inputdata_root.rglob("*.tmp")) + assert not tmp_paths, f"Found unexpected '.tmp' path(s) left behind: {tmp_paths}" + assert inner_file.read_text() == "clm2 data" + + def test_empty_string_argument_errors_and_leaves_tree_intact( + self, rimport_script, test_env, rimport_env + ): + """Test that an empty-string positional (as from an unset shell variable, e.g. + `rimport "$maybe_unset"`) errors cleanly instead of anchoring to the inputdata root + itself and running that root through the destructive replace-with-symlink path.""" + inputdata_root = test_env["inputdata_root"] + # staging_root itself need not be assigned here — the fixture already created it, and + # its mere existence is what makes dst.exists() true for rel="." — the same thing that + # turns an unset shell variable into a whole-tree rename. + + marker_file = inputdata_root / "marker.nc" + marker_file.write_text("root marker") + + # Run rimport with an empty-string positional, from inside the inputdata root itself, + # so it anchors (via _anchor_cli) to the root — the same as an unset shell variable + # expanding to "". + command = [ + sys.executable, + rimport_script, + "", + "-inputdata", + str(inputdata_root), + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + cwd=inputdata_root, + ) + + # Verify failure + assert result.returncode != 0, f"Command unexpectedly passed: {result.stdout}" + assert "directory" in result.stderr + assert "not a file" in result.stderr + + # Verify the inputdata root itself is untouched: still a real directory, not renamed, + # not replaced with a symlink, no '.tmp' sibling. + assert inputdata_root.is_dir() and not inputdata_root.is_symlink(), ( + f"inputdata root should still be a plain, non-symlink directory after the error; " + f"is_dir={inputdata_root.is_dir()} is_symlink={inputdata_root.is_symlink()}" + ) + tmp_siblings = list(inputdata_root.parent.glob(f"{inputdata_root.name}.tmp")) + assert not tmp_siblings, f"Found unexpected '.tmp' sibling(s): {tmp_siblings}" + assert marker_file.read_text() == "root marker" + + def test_check_directory_argument_reports_error_not_publishable( + self, rimport_script, test_env, rimport_env + ): + """Test that --check on a directory argument reports it as an error, rather than + misreporting the directory as already published but not linked and available for + download.""" + inputdata_root = test_env["inputdata_root"] + staging_root = test_env["staging_root"] + + subdir = inputdata_root / "lnd" / "clm2" + subdir.mkdir(parents=True) + inner_file = subdir / "data.nc" + inner_file.write_text("clm2 data") + + # Matching staging mirror, as in the destructive-path test above. + staging_mirror = staging_root / "lnd" / "clm2" + staging_mirror.mkdir(parents=True) + + command = [ + sys.executable, + rimport_script, + "clm2", + "-inputdata", + str(inputdata_root), + "--check", + ] + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + cwd=subdir.parent, + ) + + # Verify failure + assert result.returncode != 0, f"Command unexpectedly passed: {result.stdout}" + assert "directory" in result.stderr + assert "not a file" in result.stderr + + # Verify --check does NOT claim the directory is already published / downloadable + assert "already published" not in result.stdout.lower() + assert "available for download" not in result.stdout.lower() + + # Verify the tree is intact + assert subdir.is_dir() and not subdir.is_symlink() + assert not list(inputdata_root.rglob("*.tmp")) + assert inner_file.read_text() == "clm2 data" + + def _run_mixed_validity_list(self, rimport_script, test_env, rimport_env, *, check): + """Set up a --list with one valid entry (good.nc) and two entries that are invalid + in DIFFERENT ways (missing.nc, and a directory named adir), all as absolute paths in + a list file OUTSIDE the inputdata tree, then run rimport against it -- with + --check when `check` is True (which also requires deleting + RIMPORT_SKIP_USER_CHECK, since --check needs ensure_running_as() to actually run), + without it otherwise. + + Shared setup and invocation for test_mixed_validity_list_aborts_and_stages_nothing + and test_check_mode_is_gated_too_and_reports_nothing_for_valid_entry. The two tests + differ in `check` and, more importantly, in what each one's own payoff assertions + check afterward -- see each test's docstring. This helper asserts only the rc-2 and + failure-reason outcome that is IDENTICAL for both callers; those assertions still + matter (they would catch the pre-flight gate failing to fire at all) but are not what + tells the two tests apart -- the payoff assertions that make each test meaningful stay + in the tests, not here. + + Returns (result, valid_file, staging_root) for the caller to assert against. + """ + inputdata_root = test_env["inputdata_root"] + staging_root = test_env["staging_root"] + tmp_path = test_env["tmp_path"] + + valid_file = inputdata_root / "good.nc" + valid_file.write_text("good data") + + missing_file = inputdata_root / "missing.nc" + + bad_dir = inputdata_root / "adir" + bad_dir.mkdir() + + # List file OUTSIDE the tree, with absolute entries. + filelist = tmp_path / "filelist.txt" + filelist.write_text(f"{valid_file}\n{missing_file}\n{bad_dir}\n") + + command = [ + sys.executable, + rimport_script, + "-list", + str(filelist), + "-inputdata", + str(inputdata_root), + ] + if check: + # Make sure --check skips ensure_running_as() + del rimport_env["RIMPORT_SKIP_USER_CHECK"] + command.append("--check") + + result = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + env=rimport_env, + ) + + # Verify failure: rc 2, all reasons present, correct "N of M" count. Identical for + # both callers; not what either test discriminates on. + assert result.returncode == 2, f"Command unexpectedly passed: {result.stdout}" + assert "2 of 3 file(s) failed pre-flight validation" in result.stderr + assert f"source not found: {missing_file}" in result.stderr + assert f"source is a directory, not a file: {bad_dir}" in result.stderr + + return result, valid_file, staging_root + + def test_mixed_validity_list_aborts_and_stages_nothing( + self, rimport_script, test_env, rimport_env + ): + """Test the pre-flight gate end to end: a --list with one valid entry and two entries + that are invalid in DIFFERENT ways (missing, and a directory) aborts the whole batch + with rc 2, reports every failure reason, gets the "N of M" count right, and — the + assertion that matters most — never stages or relinks the valid entry. + + The list file lives OUTSIDE the inputdata tree with absolute entries.""" + _result, valid_file, staging_root = self._run_mixed_validity_list( + rimport_script, test_env, rimport_env, check=False + ) + + # Verify the valid file was NOT staged and NOT turned into a symlink + assert not (staging_root / "good.nc").exists() + assert not valid_file.is_symlink() + assert valid_file.read_text() == "good data" + + # Verify nothing at all was staged + assert not any(staging_root.rglob("*")) + + def test_check_mode_is_gated_too_and_reports_nothing_for_valid_entry( + self, rimport_script, test_env, rimport_env + ): + """Test that --check is subject to the same pre-flight gate as a real run: a mix of + valid and invalid entries aborts with rc 2 and the valid entry's status is NOT + reported. + + This pins a deliberate design decision: uniform abort, chosen over per-file --check + reporting, even though it means a --check run tells you nothing about the files that + would have been fine. A future reader should not silently "fix" this into per-file + --check reporting -- that would be a deliberate change in behavior, not a bug fix.""" + result, valid_file, staging_root = self._run_mixed_validity_list( + rimport_script, test_env, rimport_env, check=True + ) + + # Verify the valid entry's check status is NOT reported: --check never gets to run + # per-file, so neither the "already published" nor "not already published" messages + # appear anywhere, for any file. + assert "already published" not in result.stdout.lower() + assert "not already published" not in result.stdout.lower() + assert result.stdout == "" + + # Verify nothing was staged + assert not any(staging_root.rglob("*")) + assert not valid_file.is_symlink() diff --git a/tests/rimport/test_get_files_to_process.py b/tests/rimport/test_get_files_to_process.py index 6fc13b3..e709e5e 100644 --- a/tests/rimport/test_get_files_to_process.py +++ b/tests/rimport/test_get_files_to_process.py @@ -2,9 +2,11 @@ Tests for get_files_to_process function in rimport script. """ +import logging import os import importlib.util from importlib.machinery import SourceFileLoader +from pathlib import Path # pylint: disable=too-many-arguments,too-many-positional-arguments @@ -25,7 +27,7 @@ class TestGetRelnamesToProcess: """Test suite for get_relnames_to_process() function.""" - def test_single_file_relpath(self, tmp_path): + def test_single_file_relpath(self, tmp_path, monkeypatch): """Test giving it a single file by its relative path""" # Setup inputdata_root = tmp_path / "inputdata" @@ -37,6 +39,9 @@ def test_single_file_relpath(self, tmp_path): test_file = inputdata_root / filename test_file.write_text("abc123") + # A relative name always anchors to cwd + monkeypatch.chdir(inputdata_root) + # Run files_to_process, result = rimport.get_files_to_process( file=filename, @@ -46,7 +51,7 @@ def test_single_file_relpath(self, tmp_path): # Verify assert result == 0 - assert files_to_process == [filename] + assert files_to_process == [str(inputdata_root.resolve() / filename)] def test_single_file_abspath(self, tmp_path): """Test giving it a single file by its absolute path""" @@ -71,8 +76,8 @@ def test_single_file_abspath(self, tmp_path): assert result == 0 assert files_to_process == [test_file] - def test_filelist_relpath_with_relpaths(self, tmp_path): - """Test giving it a file list by its relative path, containing relative paths""" + def test_filelist_relpath_with_abspaths(self, tmp_path): + """Test giving it a file list by its relative path, containing absolute paths""" # Setup inputdata_root = tmp_path / "inputdata" inputdata_root.mkdir() @@ -81,9 +86,9 @@ def test_filelist_relpath_with_relpaths(self, tmp_path): filenames = [] for i in range(2): - filename = f"test{i}.txt" - filenames.append(filename) - (inputdata_root / filename).write_text("def567") + filename = inputdata_root / f"test{i}.txt" + filenames.append(str(filename)) + filename.write_text("def567") filelist = tmp_path / "file_list.txt" filelist.write_text("\n".join(filenames), encoding="utf8") @@ -100,8 +105,8 @@ def test_filelist_relpath_with_relpaths(self, tmp_path): assert result == 0 assert files_to_process == filenames - def test_filelist_abspath_with_relpaths(self, tmp_path): - """Test giving it a file list by its absolute path, containing relative paths""" + def test_filelist_abspath_with_abspaths(self, tmp_path): + """Test giving it a file list by its absolute path, containing absolute paths""" # Setup inputdata_root = tmp_path / "inputdata" inputdata_root.mkdir() @@ -110,9 +115,9 @@ def test_filelist_abspath_with_relpaths(self, tmp_path): filenames = [] for i in range(2): - filename = f"test{i}.txt" - filenames.append(filename) - (inputdata_root / filename).write_text("def567") + filename = inputdata_root / f"test{i}.txt" + filenames.append(str(filename)) + filename.write_text("def567") filelist = tmp_path / "file_list.txt" filelist.write_text("\n".join(filenames), encoding="utf8") @@ -128,42 +133,125 @@ def test_filelist_abspath_with_relpaths(self, tmp_path): assert result == 0 assert files_to_process == filenames - def test_filelist_relpath_with_abspaths(self, tmp_path): - """Test giving it a file list by its relative path, containing absolute paths""" + def test_list_inside_tree_relative_entries_anchored_to_list_dir(self, tmp_path): + """Test that relative entries in a list file inside the tree anchor to the list file's + own directory, not the inputdata root""" # Setup inputdata_root = tmp_path / "inputdata" - inputdata_root.mkdir() - staging_root = tmp_path / "staging" - staging_root.mkdir() + list_dir = inputdata_root / "lnd" + list_dir.mkdir(parents=True) - filenames = [] - for i in range(2): - filename = inputdata_root / f"test{i}.txt" - filenames.append(str(filename)) - filename.write_text("def567") + filenames = ["clm2/file1.nc", "file2.nc"] + filelist = list_dir / "filelist.txt" + filelist.write_text("\n".join(filenames), encoding="utf8") - filelist = tmp_path / "file_list.txt" + # Run + files_to_process, result = rimport.get_files_to_process( + file=None, + filelist=filelist, + items_to_process=None, + ) + + # Verify + assert result == 0 + list_dir_resolved = list_dir.resolve() + assert files_to_process == [str(list_dir_resolved / f) for f in filenames] + + def test_list_relative_entries_anchor_to_list_dir_not_cwd(self, tmp_path, monkeypatch): + """Test that relative list entries anchor to the list file's own directory even when + the cwd is inside the tree at a DIFFERENT location. This is the discriminating setup: + every other list test runs with cwd outside the tree, where cwd-anchoring and + list-dir-anchoring agree and so can't tell the two schemes apart.""" + # Setup + inputdata_root = tmp_path / "inputdata" + atm_dir = inputdata_root / "atm" + list_dir = inputdata_root / "lnd" + atm_dir.mkdir(parents=True) + list_dir.mkdir(parents=True) + + # Real file, alongside the list file's own subtree + real_file = list_dir / "clm2" / "file1.nc" + real_file.parent.mkdir(parents=True) + real_file.write_text("real data") + + # Decoy at the cwd-anchored location: a cwd-anchoring regression would resolve here + # instead, giving a wrong-file failure rather than a merely-missing-file one. + decoy_file = atm_dir / "clm2" / real_file.name + decoy_file.parent.mkdir(parents=True) + decoy_file.write_text("decoy data") + assert decoy_file.read_text() != real_file.read_text() + + filelist = list_dir / "filelist.txt" + filelist.write_text(f"clm2/{real_file.name}\n", encoding="utf8") + + # cwd inside the tree, but at a different location than the list file + monkeypatch.chdir(atm_dir) + + # Run + files_to_process, result = rimport.get_files_to_process( + file=None, + filelist=filelist, + items_to_process=None, + ) + + # Verify + assert result == 0 + assert files_to_process == [str(real_file.resolve())] + + def test_list_at_root_relative_entries_anchored_to_root(self, tmp_path): + """Test that a list file located at the inputdata root itself (root counts as inside the + tree) anchors relative entries to the root""" + # Setup + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + + filenames = ["test0.txt", "test1.txt"] + filelist = inputdata_root / "filelist.txt" filelist.write_text("\n".join(filenames), encoding="utf8") - filelist_relpath = os.path.relpath(filelist) # Run files_to_process, result = rimport.get_files_to_process( file=None, - filelist=filelist_relpath, + filelist=filelist, items_to_process=None, ) # Verify assert result == 0 - assert files_to_process == filenames + root_resolved = inputdata_root.resolve() + assert files_to_process == [str(root_resolved / f) for f in filenames] + + def test_list_outside_tree_relative_entry_anchors_to_list_dir(self, tmp_path, monkeypatch): + """Test that a relative entry in a list file OUTSIDE the tree anchors to the list + file's own directory rather than erroring. Discriminating setup: cwd is neither the + list dir nor the inputdata root, so the assertion can distinguish list-dir-anchoring + from cwd-anchoring and from root-anchoring.""" + # Setup + list_dir = tmp_path / "outside" / "listdir" + list_dir.mkdir(parents=True) + filelist = list_dir / "filelist.txt" + filelist.write_text("relative_file.nc\n", encoding="utf8") - def test_filelist_abspath_with_abspaths(self, tmp_path): - """Test giving it a file list by its absolute path, containing absolute paths""" + elsewhere = tmp_path / "outside" / "elsewhere" + elsewhere.mkdir(parents=True) + monkeypatch.chdir(elsewhere) + + # Run + files_to_process, result = rimport.get_files_to_process( + file=None, + filelist=filelist, + items_to_process=None, + ) + + # Verify + assert result == 0 + assert files_to_process == [str((list_dir / "relative_file.nc").resolve())] + + def test_list_outside_tree_absolute_entries_ok(self, tmp_path): + """Test that a list file outside the tree still works when all entries are absolute""" # Setup inputdata_root = tmp_path / "inputdata" inputdata_root.mkdir() - staging_root = tmp_path / "staging" - staging_root.mkdir() filenames = [] for i in range(2): @@ -185,7 +273,7 @@ def test_filelist_abspath_with_abspaths(self, tmp_path): assert result == 0 assert files_to_process == filenames - def test_filelist_not_found(self): + def test_filelist_not_found(self, tmp_path): """Test giving it a file list that doesn't exist""" filelist = "bsfearirn" assert not os.path.exists(filelist) @@ -234,7 +322,7 @@ def test_items_to_process_abspaths(self, tmp_path): assert result == 0 assert files_to_process == filenames - def test_items_to_process_relpaths(self, tmp_path): + def test_items_to_process_relpaths(self, tmp_path, monkeypatch): """Test giving it a list of relative paths in items_to_process""" # Setup inputdata_root = tmp_path / "inputdata" @@ -246,6 +334,9 @@ def test_items_to_process_relpaths(self, tmp_path): filenames.append(os.path.basename(filename)) filename.write_text("def567") + # A relative name always anchors to cwd + monkeypatch.chdir(inputdata_root) + # Run files_to_process, result = rimport.get_files_to_process( file=None, @@ -255,9 +346,9 @@ def test_items_to_process_relpaths(self, tmp_path): # Verify assert result == 0 - assert files_to_process == filenames + assert files_to_process == [str(inputdata_root.resolve() / f) for f in filenames] - def test_items_to_process_mixpaths(self, tmp_path): + def test_items_to_process_mixpaths(self, tmp_path, monkeypatch): """Test giving it a list of absolute and relative paths in items_to_process""" # Setup inputdata_root = tmp_path / "inputdata" @@ -274,6 +365,9 @@ def test_items_to_process_mixpaths(self, tmp_path): filename.write_text("def567") assert len(filenames) == 4 + # A relative name always anchors to cwd; absolute names are unaffected + monkeypatch.chdir(inputdata_root) + # Run files_to_process, result = rimport.get_files_to_process( file=None, @@ -283,18 +377,26 @@ def test_items_to_process_mixpaths(self, tmp_path): # Verify assert result == 0 - assert files_to_process == filenames + assert files_to_process == [str(inputdata_root.resolve() / f) for f in filenames[:2]] + ( + filenames[2:] + ) - def test_single_file_and_list(self, tmp_path): - """Test giving it a single file by its relative path""" + def test_single_file_and_list(self, tmp_path, monkeypatch): + """Test giving it a single file by its relative path together with a list file. + Discriminating setup: cwd is a SUBDIRECTORY of inputdata_root, distinct from the list + file's own directory (inputdata_root itself), so the test pins cwd-anchoring for the + relative `file` and list-dir-anchoring for the list entries at once, and can tell the + two apart.""" # Setup inputdata_root = tmp_path / "inputdata" inputdata_root.mkdir() staging_root = tmp_path / "staging" staging_root.mkdir() + subdir = inputdata_root / "sub" + subdir.mkdir() filename = "test.nc" - test_file = inputdata_root / filename + test_file = subdir / filename test_file.write_text("abc123") filenames = [] @@ -303,9 +405,12 @@ def test_single_file_and_list(self, tmp_path): filenames.append(f) (inputdata_root / f).write_text("def567") - filelist = tmp_path / "file_list.txt" + filelist = inputdata_root / "file_list.txt" filelist.write_text("\n".join(filenames), encoding="utf8") + # cwd is inside the tree, but at the subdir, not the list file's own directory + monkeypatch.chdir(subdir) + # Run files_to_process, result = rimport.get_files_to_process( file=filename, @@ -315,9 +420,11 @@ def test_single_file_and_list(self, tmp_path): # Verify assert result == 0 - assert files_to_process == [filename] + filenames + assert files_to_process == [str(subdir.resolve() / filename)] + [ + str(inputdata_root.resolve() / f) for f in filenames + ] - def test_single_or_filelist_or_list_required(self): + def test_single_or_filelist_or_list_required(self, tmp_path): """Test that at least one of file, filelist, items_to_process is required""" # Run files_to_process, result = rimport.get_files_to_process( @@ -329,3 +436,213 @@ def test_single_or_filelist_or_list_required(self): # Verify assert result == 2 assert files_to_process is None + + def test_cli_file_relative_cwd_inside_tree(self, tmp_path, monkeypatch): + """Test that a relative --file name anchors to cwd when cwd is inside the tree""" + inputdata_root = tmp_path / "inputdata" + subdir = inputdata_root / "sub" + subdir.mkdir(parents=True) + monkeypatch.chdir(subdir) + cwd = Path.cwd().resolve() + + filename = "test.nc" + + # Run + files_to_process, result = rimport.get_files_to_process( + file=filename, + filelist=None, + items_to_process=None, + ) + + # Verify + assert result == 0 + assert files_to_process == [str(cwd / filename)] + + def test_cli_items_relative_cwd_inside_tree(self, tmp_path, monkeypatch): + """Test that relative items_to_process names anchor to cwd when cwd is inside the tree""" + inputdata_root = tmp_path / "inputdata" + subdir = inputdata_root / "sub" + subdir.mkdir(parents=True) + monkeypatch.chdir(subdir) + cwd = Path.cwd().resolve() + + filenames = ["test0.txt", "test1.txt"] + + # Run + files_to_process, result = rimport.get_files_to_process( + file=None, + filelist=None, + items_to_process=filenames, + ) + + # Verify + assert result == 0 + assert files_to_process == [str(cwd / f) for f in filenames] + + def test_cli_relative_cwd_outside_tree_still_anchors_to_cwd(self, tmp_path, monkeypatch): + """Test that a relative --file name anchors to cwd even when cwd is outside the + inputdata tree: there is no inside/outside distinction, one rule applies everywhere, + with no root-relative fallback.""" + outside = tmp_path / "outside" + outside.mkdir() + monkeypatch.chdir(outside) + + filename = "test.nc" + + # Run + files_to_process, result = rimport.get_files_to_process( + file=filename, + filelist=None, + items_to_process=None, + ) + + # Verify + assert result == 0 + assert files_to_process == [str(outside.resolve() / filename)] + + def test_cli_cwd_equals_root_anchors_to_root(self, tmp_path, monkeypatch): + """Test that cwd == inputdata root counts as inside the tree""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + monkeypatch.chdir(inputdata_root) + cwd = Path.cwd().resolve() + + filename = "test.nc" + + # Run + files_to_process, result = rimport.get_files_to_process( + file=filename, + filelist=None, + items_to_process=None, + ) + + # Verify + assert result == 0 + assert files_to_process == [str(cwd / filename)] + + def test_cli_absolute_unchanged_cwd_inside_tree(self, tmp_path, monkeypatch): + """Test that an absolute --file name is left unchanged even when cwd is inside the tree""" + inputdata_root = tmp_path / "inputdata" + subdir = inputdata_root / "sub" + subdir.mkdir(parents=True) + monkeypatch.chdir(subdir) + + abs_file = str(inputdata_root / "other" / "test.nc") + + # Run + files_to_process, result = rimport.get_files_to_process( + file=abs_file, + filelist=None, + items_to_process=None, + ) + + # Verify + assert result == 0 + assert files_to_process == [abs_file] + + def test_cli_cwd_inside_tree_via_symlink(self, tmp_path, monkeypatch): + """Test that a relative name anchors to the real (resolved) cwd when cwd was reached + through a symlink into the tree.""" + inputdata_root = tmp_path / "inputdata" + real_sub = inputdata_root / "sub" + real_sub.mkdir(parents=True) + link = tmp_path / "link" + link.symlink_to(real_sub) + monkeypatch.chdir(link) + + filename = "test.nc" + + # Run + files_to_process, result = rimport.get_files_to_process( + file=filename, + filelist=None, + items_to_process=None, + ) + + # Verify + assert result == 0 + assert files_to_process == [str(real_sub.resolve() / filename)] + + def test_deleted_cwd_with_absolute_names_still_works(self, tmp_path, monkeypatch, caplog): + """Test that a deleted cwd does not raise: absolute names are returned unchanged since + cwd is irrelevant to resolving them. Simulates a deleted cwd for real (not mocked): chdir + into a directory, then remove it out from under the process, which raises + FileNotFoundError from Path.cwd() on this platform.""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + abs_file = str(inputdata_root / "test.nc") + + deleted_dir = tmp_path / "deleted" + deleted_dir.mkdir() + monkeypatch.chdir(deleted_dir) + deleted_dir.rmdir() + + # Run + with caplog.at_level(logging.WARNING): + files_to_process, result = rimport.get_files_to_process( + file=abs_file, + filelist=None, + items_to_process=None, + ) + + # Verify + assert result == 0 + assert files_to_process == [abs_file] + # No warning: the cwd being undeterminable doesn't matter for absolute names + assert "working directory" not in caplog.text.lower() + + def test_deleted_cwd_with_relative_name_errors(self, tmp_path, monkeypatch, caplog): + """Test that a deleted cwd is a fatal error for relative names: with no root-relative + fallback, there is nothing left to anchor a relative name against. Confirms rc 2, + files_to_process is None, and that the error message names the offending relative + name.""" + filename = "test.nc" + + deleted_dir = tmp_path / "deleted" + deleted_dir.mkdir() + monkeypatch.chdir(deleted_dir) + deleted_dir.rmdir() + + # Run + with caplog.at_level(logging.ERROR): + files_to_process, result = rimport.get_files_to_process( + file=filename, + filelist=None, + items_to_process=None, + ) + + # Verify + assert result == 2 + assert files_to_process is None + assert filename in caplog.text + assert "working directory" in caplog.text.lower() + + def test_deleted_cwd_with_multiple_relative_names_reports_all( + self, tmp_path, monkeypatch, caplog + ): + """Test that when a deleted cwd leaves several relative names unresolvable, the error + message names ALL of them, not just the first -- failing before doing anything and + naming every offending input at once gives the user everything they need to fix in one + pass, rather than fixing inputs one at a time across repeated runs.""" + filename = "test.nc" + item_names = ["a.txt", "b.txt"] + + deleted_dir = tmp_path / "deleted" + deleted_dir.mkdir() + monkeypatch.chdir(deleted_dir) + deleted_dir.rmdir() + + # Run + with caplog.at_level(logging.ERROR): + files_to_process, result = rimport.get_files_to_process( + file=filename, + filelist=None, + items_to_process=item_names, + ) + + # Verify + assert result == 2 + assert files_to_process is None + assert filename in caplog.text + for item_name in item_names: + assert item_name in caplog.text diff --git a/tests/rimport/test_main.py b/tests/rimport/test_main.py index 3451199..04ba444 100644 --- a/tests/rimport/test_main.py +++ b/tests/rimport/test_main.py @@ -30,6 +30,7 @@ class TestMain: """Test suite for main() function.""" + @patch.object(rimport, "validate_source_path") @patch.object(rimport, "stage_data") @patch.object(rimport, "get_staging_root") @patch.object(rimport, "normalize_paths") @@ -40,6 +41,7 @@ def test_single_file_success( mock_normalize_paths, mock_get_staging_root, mock_stage_data, + mock_validate_source_path, tmp_path, caplog, ): @@ -53,19 +55,27 @@ def test_single_file_success( mock_get_staging_root.return_value = staging_root test_file = inputdata_root / "test.nc" mock_normalize_paths.return_value = [test_file] + # test_file doesn't exist on disk; bypass pre-flight so this test still exercises only + # main()'s control flow, not validate_source_path's real checks. + mock_validate_source_path.return_value = None # Run - result = rimport.main(["-file", "test.nc", "-inputdata", str(inputdata_root)]) + # Absolute -file removes cwd coupling entirely: get_files_to_process runs for real here + # (unmocked), and a relative name would anchor to the pytest invocation dir, not tmp_path. + result = rimport.main( + ["-file", str(test_file), "-inputdata", str(inputdata_root)] + ) # Verify assert result == 0 - mock_normalize_paths.assert_called_once_with(inputdata_root, ["test.nc"]) + mock_normalize_paths.assert_called_once_with(inputdata_root, [str(test_file)]) check = False mock_stage_data.assert_called_once_with( test_file, inputdata_root, staging_root, check ) assert "No need to run relink.py" in caplog.text + @patch.object(rimport, "validate_source_path") @patch.object(rimport, "stage_data") @patch.object(rimport, "get_staging_root") @patch.object(rimport, "normalize_paths") @@ -78,6 +88,7 @@ def test_file_list_success( mock_normalize_paths, mock_get_staging_root, mock_stage_data, + mock_validate_source_path, tmp_path, ): """Test main() logic flow when a file list stages successfully.""" @@ -86,7 +97,7 @@ def test_file_list_success( inputdata_root.mkdir() staging_root = tmp_path / "staging" staging_root.mkdir() - filelist = tmp_path / "files.txt" + filelist = inputdata_root / "files.txt" filelist.write_text("file1.nc\nfile2.nc\n") mock_get_staging_root.return_value = staging_root @@ -94,6 +105,9 @@ def test_file_list_success( file1 = inputdata_root / "file1.nc" file2 = inputdata_root / "file2.nc" mock_normalize_paths.return_value = [file1, file2] + # Neither file exists on disk; bypass pre-flight so this test still exercises only + # main()'s control flow, not validate_source_path's real checks. + mock_validate_source_path.return_value = None # Run result = rimport.main( @@ -104,7 +118,11 @@ def test_file_list_success( assert result == 0 mock_read_filelist.assert_called_once_with(filelist) mock_normalize_paths.assert_called_once_with( - inputdata_root, ["file1.nc", "file2.nc"] + inputdata_root, + [ + str(inputdata_root.resolve() / "file1.nc"), + str(inputdata_root.resolve() / "file2.nc"), + ], ) assert mock_stage_data.call_count == 2 check = False @@ -115,6 +133,7 @@ def test_file_list_success( ] ) + @patch.object(rimport, "validate_source_path") @patch.object(rimport, "stage_data") @patch.object(rimport, "get_staging_root") @patch.object(rimport, "normalize_paths") @@ -125,6 +144,7 @@ def test_stage_data_exception_handling( mock_normalize_paths, _mock_get_staging_root, mock_stage_data, + mock_validate_source_path, tmp_path, capsys, ): @@ -137,6 +157,9 @@ def test_stage_data_exception_handling( file2 = inputdata_root / "file2.nc" file3 = inputdata_root / "file3.nc" mock_normalize_paths.return_value = [file1, file2, file3] + # None of the files exist on disk; bypass pre-flight so this test still exercises only + # main()'s per-file error handling, not validate_source_path's real checks. + mock_validate_source_path.return_value = None # Make stage_data fail for file2 but succeed for others def stage_data_side_effect(src, *_args, **_kwargs): @@ -223,6 +246,7 @@ def test_requires_file_or_filelist(self, _mock_ensure_running_as, tmp_path, caps captured = capsys.readouterr() assert "At least one of --file or --filelist is required" in captured.err + @patch.object(rimport, "validate_source_path") @patch.object(rimport, "stage_data") @patch.object(rimport, "get_staging_root") @patch.object(rimport, "normalize_paths") @@ -233,6 +257,7 @@ def test_check_mode_calls( mock_normalize_paths, mock_get_staging_root, mock_stage_data, + mock_validate_source_path, tmp_path, caplog, ): @@ -245,6 +270,9 @@ def test_check_mode_calls( mock_get_staging_root.return_value = staging_root test_file = inputdata_root / "test.nc" mock_normalize_paths.return_value = [test_file] + # test_file doesn't exist on disk; bypass pre-flight (which runs in --check mode too) + # so this test still exercises only main()'s control flow. + mock_validate_source_path.return_value = None result = rimport.main( ["-file", "test.nc", "-inputdata", str(inputdata_root), "--check"] @@ -261,6 +289,7 @@ def test_check_mode_calls( # Message about relink.py should not have been printed assert "No need to run relink.py" not in caplog.text + @patch.object(rimport, "validate_source_path") @patch.object(rimport, "stage_data") @patch.object(rimport, "get_staging_root") @patch.object(rimport, "normalize_paths") @@ -271,6 +300,7 @@ def test_skip_user_check_env_var( mock_normalize_paths, _mock_get_staging_root, _mock_stage, + mock_validate_source_path, tmp_path, monkeypatch, ): @@ -282,6 +312,9 @@ def test_skip_user_check_env_var( test_file = inputdata_root / "test.nc" mock_normalize_paths.return_value = [test_file] + # test_file doesn't exist on disk; bypass pre-flight so this test still exercises only + # the user-check skip logic. + mock_validate_source_path.return_value = None result = rimport.main(["-file", "test.nc", "-inputdata", str(inputdata_root)]) @@ -289,6 +322,7 @@ def test_skip_user_check_env_var( # ensure_running_as should NOT be called when env var is set mock_ensure_running_as.assert_not_called() + @patch.object(rimport, "validate_source_path") @patch.object(rimport, "stage_data") @patch.object(rimport, "get_staging_root") @patch.object(rimport, "normalize_paths") @@ -299,6 +333,7 @@ def test_prints_file_path_before_processing( mock_normalize_paths, _mock_get_staging_root, _mock_stage, + mock_validate_source_path, tmp_path, capsys, ): @@ -308,6 +343,9 @@ def test_prints_file_path_before_processing( file1 = inputdata_root / "file1.nc" file2 = inputdata_root / "file2.nc" mock_normalize_paths.return_value = [file1, file2] + # Neither file exists on disk; bypass pre-flight so this test still exercises only the + # per-file print, not validate_source_path's real checks. + mock_validate_source_path.return_value = None result = rimport.main(["-file", "test.nc", "-inputdata", str(inputdata_root)]) @@ -317,6 +355,7 @@ def test_prints_file_path_before_processing( assert f"'{file1}':" in captured.out assert f"'{file2}':" in captured.out + @patch.object(rimport, "validate_source_path") @patch.object(rimport, "stage_data") @patch.object(rimport, "get_staging_root") @patch.object(rimport, "normalize_paths") @@ -327,6 +366,7 @@ def test_multiple_errors_returns_1( mock_normalize_paths, _mock_get_staging_root, mock_stage_data, + mock_validate_source_path, tmp_path, ): """Test that main() returns 1 when multiple files fail.""" @@ -337,6 +377,9 @@ def test_multiple_errors_returns_1( file2 = inputdata_root / "file2.nc" file3 = inputdata_root / "file3.nc" mock_normalize_paths.return_value = [file1, file2, file3] + # None of the files exist on disk; bypass pre-flight so this test still exercises the + # per-file loop's runtime-error handling, not validate_source_path's real checks. + mock_validate_source_path.return_value = None # Make all files fail mock_stage_data.side_effect = RuntimeError("Test error") @@ -346,6 +389,7 @@ def test_multiple_errors_returns_1( assert result == 1 assert mock_stage_data.call_count == 3 + @patch.object(rimport, "validate_source_path") @patch.object(rimport, "stage_data") @patch.object(rimport, "get_staging_root") @patch.object(rimport, "normalize_paths") @@ -356,6 +400,7 @@ def test_error_counter_increments_correctly( mock_normalize_paths, _mock_get_staging_root, mock_stage_data, + mock_validate_source_path, tmp_path, capsys, ): @@ -365,6 +410,9 @@ def test_error_counter_increments_correctly( files = [inputdata_root / f"file{i}.nc" for i in range(5)] mock_normalize_paths.return_value = files + # None of the files exist on disk; bypass pre-flight so this test still exercises only + # the per-file error counter, not validate_source_path's real checks. + mock_validate_source_path.return_value = None # Make files 1 and 3 fail def stage_data_side_effect(src, *_args, **_kwargs): @@ -457,3 +505,56 @@ def test_error_if_file_newly_published_but_relink_fails( captured = capsys.readouterr() assert "rimport: error processing" in captured.err assert "Error relinking during rimport" in captured.err + + @patch.object(rimport, "stage_data") + @patch.object(rimport, "get_staging_root") + @patch.object(rimport, "ensure_running_as") + def test_preflight_gate_rejects_whole_batch_and_never_calls_stage_data( + self, + _mock_ensure_running_as, + mock_get_staging_root, + mock_stage_data, + tmp_path, + capsys, + ): + """Test main()'s pre-flight gate: a batch with a mix of valid and invalid paths returns + 2, logs every failure, and never calls stage_data — not even for the valid path. + + Unlike the other main() tests in this file, this one does NOT mock + validate_source_path (or normalize_paths): it lets the real pre-flight gate run + against real files, so it is actually exercising the gate rather than assuming it + works. + """ + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + staging_root = tmp_path / "staging" + staging_root.mkdir() + mock_get_staging_root.return_value = staging_root + + valid = inputdata_root / "good.nc" + valid.write_text("data") + missing = inputdata_root / "missing.nc" + bad_dir = inputdata_root / "adir" + bad_dir.mkdir() + + result = rimport.main( + [ + "-inputdata", + str(inputdata_root), + str(valid), + str(missing), + str(bad_dir), + ] + ) + + assert result == 2 + mock_stage_data.assert_not_called() + + captured = capsys.readouterr() + assert "2 of 3 file(s) failed pre-flight validation" in captured.err + assert f"source not found: {missing}" in captured.err + assert f"source is a directory, not a file: {bad_dir}" in captured.err + + # The valid file was never staged or turned into a symlink. + assert not (staging_root / "good.nc").exists() + assert not valid.is_symlink() diff --git a/tests/rimport/test_validate_source_path.py b/tests/rimport/test_validate_source_path.py new file mode 100644 index 0000000..7f136b3 --- /dev/null +++ b/tests/rimport/test_validate_source_path.py @@ -0,0 +1,146 @@ +""" +Tests for validate_source_path() function in rimport script. +""" + +import os +import importlib.util +from importlib.machinery import SourceFileLoader + + +# Import rimport module from file without .py extension +rimport_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "rimport", +) +loader = SourceFileLoader("rimport", rimport_path) +spec = importlib.util.spec_from_loader("rimport", loader) +if spec is None: + raise ImportError(f"Could not create spec for rimport from {rimport_path}") +rimport = importlib.util.module_from_spec(spec) +# Don't add to sys.modules to avoid conflict with other test files +loader.exec_module(rimport) + + +def test_ok_regular_file_under_root(tmp_path): + """A valid regular file under the inputdata root returns None.""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + staging_root = tmp_path / "staging" + staging_root.mkdir() + + src = inputdata_root / "file.nc" + src.write_text("data") + + assert rimport.validate_source_path(src, inputdata_root, staging_root) is None + + +def test_ok_already_published_symlink_is_not_a_failure(tmp_path): + """DANGEROUS CASE: a live symlink whose target resolves under staging_root is the normal + state of a file that has already been published and linked by a previous run. This must + return None, not an exception — otherwise re-running rimport over an already-published + tree would report every published file as a pre-flight failure.""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + staging_root = tmp_path / "staging" + staging_root.mkdir() + + real_file = staging_root / "real_file.nc" + real_file.write_text("data") + src = inputdata_root / "link.nc" + src.symlink_to(real_file) + + assert rimport.validate_source_path(src, inputdata_root, staging_root) is None + + +def test_error_broken_symlink(tmp_path): + """A symlink whose target does not exist returns a RuntimeError, unraised.""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + staging_root = tmp_path / "staging" + staging_root.mkdir() + + src = inputdata_root / "broken_link.nc" + src.symlink_to(tmp_path / "nonexistent.nc") + + result = rimport.validate_source_path(src, inputdata_root, staging_root) + assert isinstance(result, RuntimeError) + assert "Source is a broken symlink" in str(result) + + +def test_error_live_symlink_target_outside_staging(tmp_path): + """A live symlink whose target resolves outside staging_root returns a RuntimeError, + unraised.""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + staging_root = tmp_path / "staging" + staging_root.mkdir() + + real_file = tmp_path / "real_file.nc" + real_file.write_text("data") + src = inputdata_root / "link.nc" + src.symlink_to(real_file) + + result = rimport.validate_source_path(src, inputdata_root, staging_root) + assert isinstance(result, RuntimeError) + assert "outside staging directory" in str(result) + + +def test_error_missing_file(tmp_path): + """A path that does not exist returns a FileNotFoundError, unraised.""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + staging_root = tmp_path / "staging" + staging_root.mkdir() + + src = inputdata_root / "nonexistent.nc" + + result = rimport.validate_source_path(src, inputdata_root, staging_root) + assert isinstance(result, FileNotFoundError) + assert "source not found" in str(result) + + +def test_error_directory(tmp_path): + """A directory (not a symlink) returns a RuntimeError, unraised.""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + staging_root = tmp_path / "staging" + staging_root.mkdir() + + src = inputdata_root / "adir" + src.mkdir() + + result = rimport.validate_source_path(src, inputdata_root, staging_root) + assert isinstance(result, RuntimeError) + assert "source is a directory, not a file" in str(result) + + +def test_error_file_outside_inputdata_root(tmp_path): + """A regular file outside the inputdata root returns a RuntimeError, unraised.""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + staging_root = tmp_path / "staging" + staging_root.mkdir() + + src = tmp_path / "outside" / "file.nc" + src.parent.mkdir() + src.write_text("data") + + result = rimport.validate_source_path(src, inputdata_root, staging_root) + assert isinstance(result, RuntimeError) + assert "not under inputdata root" in str(result) + + +def test_error_file_already_under_staging_directory(tmp_path): + """A regular (non-symlink) file already living under staging_root returns a RuntimeError, + unraised.""" + inputdata_root = tmp_path / "inputdata" + inputdata_root.mkdir() + staging_root = tmp_path / "staging" + staging_root.mkdir() + + src = staging_root / "file.nc" + src.write_text("data") + + result = rimport.validate_source_path(src, inputdata_root, staging_root) + assert isinstance(result, RuntimeError) + assert "already under staging directory" in str(result)