From 3d552ee1d920ff898e4438848682b702a4c9ee2c Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 21 Oct 2026 22:54:30 +0000 Subject: [PATCH 01/19] test(cli): lock down the current summary output Adds tests that compare the plain-text summary line by line for upload, validate and ErrorReport, so the summary-as-data refactors (TRUNK-22401, TRUNK-22402) can show they change nothing. Each type goes through one helper (upload_summary, validate_summary, error_summary), which is the only thing those refactors should need to change. Refs TRUNK-22399 Co-Authored-By: Claude Opus 5.5 (1M context) --- cli/src/error_report.rs | 166 +++++++- cli/src/upload_command.rs | 814 +++++++++++++++++++++++++++++++++++- cli/src/validate_command.rs | 189 ++++++++- 3 files changed, 1159 insertions(+), 10 deletions(-) diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index e26bbc60..98ea9b20 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -240,6 +240,32 @@ impl EndOutput for ErrorReport { } } +#[cfg(test)] +fn error_summary(report: &ErrorReport) -> Vec { + report + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() +} + +#[cfg(test)] +pub(crate) fn unauthorized_error() -> anyhow::Error { + let response = http::Response::builder().status(401).body("").unwrap(); + anyhow::Error::from( + reqwest::Response::from(response) + .error_for_status() + .unwrap_err(), + ) +} + +#[cfg(test)] +fn connection_refused_error() -> anyhow::Error { + anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)) + .context("Failed to create bundle upload") +} + #[test] fn adds_settings_if_domain_present() { let host = "https://api.fake-trunk.io"; @@ -264,12 +290,7 @@ fn prints_a_client_error_explanation_beneath_the_base_message() { Some(String::from("There was an unexpected error")), ); - let text = report - .output() - .unwrap() - .iter() - .map(|line| line.to_unstyled()) - .collect::>(); + let text = error_summary(&report); assert_eq!( text[..4], @@ -281,3 +302,136 @@ fn prints_a_client_error_explanation_beneath_the_base_message() { ] ); } + +#[test] +fn prints_connection_refused_beneath_the_base_message() { + let report = ErrorReport::new( + connection_refused_error(), + String::from("fake-org-slug"), + Some(String::from("There was an unexpected error")), + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "There was an unexpected error", + "", + "Unable to connect to trunk's server", + ] + ); +} + +#[test] +fn prints_connection_refused_without_a_base_message() { + let report = ErrorReport::new( + connection_refused_error(), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + ["Error Encountered", "Unable to connect to trunk's server"] + ); +} + +#[test] +fn prints_unauthorized_with_a_settings_hint() { + let report = ErrorReport::new(unauthorized_error(), String::from("fake-org-slug"), None); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "", + "", + "Unathorized access, your Trunk organization URL slug or token may be incorrect", + " Hint: You can find it under the settings page at https://app.trunk.io/fake-org-slug/settings", + ] + ); +} + +#[test] +fn prints_a_generic_error_without_a_base_message() { + let report = ErrorReport::new( + anyhow::anyhow!("--allow-forked-pr-uploads requires --test-collection-id"), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "--allow-forked-pr-uploads requires --test-collection-id", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + ] + ); +} + +#[test] +fn prints_an_interrupting_error_with_its_exit_code() { + let report = ErrorReport::new( + anyhow::anyhow!(InterruptingError::new( + "No test output files found to validate" + )), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "No test output files found to validate", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, returning exit code: 70", + ] + ); +} + +#[test] +fn prints_the_default_exit_code() { + let mut report = ErrorReport::new( + anyhow::anyhow!("something went wrong"), + String::from("fake-org-slug"), + None, + ); + report.context.exit_code = Some(exitcode::OK); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "something went wrong", + "For more help, contact us at https://slack.trunk.io/", + "", + "No errors occurred, returning default exit code: 0", + ] + ); +} + +#[test] +fn prints_any_other_exit_code() { + let mut report = ErrorReport::new( + anyhow::anyhow!("something went wrong"), + String::from("fake-org-slug"), + None, + ); + report.context.exit_code = Some(exitcode::USAGE); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "something went wrong", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, returning exit code: 64", + ] + ); +} diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index faa0a33b..9e35fc2b 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -1288,9 +1288,819 @@ impl EndOutput for UploadRunResult { #[cfg(test)] mod tests { - use superconsole::{Line, style::Attribute}; + use std::collections::{BTreeMap, HashMap}; - use super::url_span; + use api::{client::ClientErrorExplanation, urls::TestCaseGuidScope}; + use bundle::{ + BundleMetaBaseProps, BundleMetaDebugProps, BundleMetaJunitProps, TestCollectionProps, + }; + use context::{ + junit::validator::{ + JunitReportValidation, JunitReportValidationIssueSubOptimal, + JunitTestCaseValidationIssueInvalid, JunitTestCaseValidationIssueSubOptimal, + JunitValidationIssue, JunitValidationIssueType, + }, + repo::{BundleRepo, RepoUrlParts}, + }; + use pretty_assertions::assert_eq; + + use super::*; + use crate::{context_quarantine::QuarantineFetchStatus, error_report::unauthorized_error}; + + const BUNDLE_UPLOAD_ID: &str = "4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d"; + const UPLOAD_ERROR_MESSAGE: &str = + "There was an unexpected error that occurred while uploading test results"; + + fn upload_summary(result: &UploadRunResult) -> Vec { + result + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() + } + + fn repo() -> RepoUrlParts { + RepoUrlParts { + host: String::from("github.com"), + owner: String::from("trunk-io"), + name: String::from("analytics-cli"), + } + } + + fn upload_result(num_tests: usize) -> UploadRunResult { + let mut quarantine_context = QuarantineContext::skip_fetch(Vec::new()); + quarantine_context.repo = repo(); + quarantine_context.org_url_slug = String::from("test-org"); + UploadRunResult { + error_report: None, + quarantine_context, + meta: BundleMeta { + base_props: BundleMetaBaseProps { + version: String::new(), + cli_version: String::new(), + org: String::from("test-org"), + test_collection: None, + repo: BundleRepo { + repo: repo(), + ..Default::default() + }, + bundle_upload_id: String::from(BUNDLE_UPLOAD_ID), + tags: Vec::new(), + file_sets: Vec::new(), + envs: HashMap::new(), + upload_time_epoch: 0, + test_command: None, + os_info: None, + quarantined_tests: Vec::new(), + codeowners: None, + use_uncloned_repo: None, + quarantine_resolution_mode: QuarantineResolutionMode::default(), + }, + junit_props: BundleMetaJunitProps { + num_files: 1, + num_tests, + }, + debug_props: BundleMetaDebugProps { + command_line: String::new(), + trunk_envs: HashMap::new(), + }, + bundle_upload_id_v2: String::new(), + variant: None, + internal_bundled_file: None, + failed_tests: Vec::new(), + }, + validations: JunitReportValidations::new(BTreeMap::new()), + validation_report: ValidationReport::Limited, + show_failure_messages: false, + test_collection_short_id: None, + hide_test_collection_links: false, + api_address: String::from("https://api.trunk.io"), + guid_scope: None, + } + } + + fn in_collection(mut result: UploadRunResult) -> UploadRunResult { + result.test_collection_short_id = Some(String::from("abc123")); + result.meta.base_props.test_collection = Some(TestCollectionProps { + short_id: String::from("abc123"), + bundle_meta_id: String::from("0b6f4c1e-8a7d-4e2b-9c3f-5d1a2b3c4d5e"), + bundle_meta_created_at: String::from("2026-10-21T12:00:00Z"), + }); + result + } + + fn failure(name: &str, parent_name: &str, file: Option<&str>) -> Test { + Test { + name: String::from(name), + parent_name: String::from(parent_name), + class_name: None, + file: file.map(String::from), + id: format!("{name}-id"), + timestamp_millis: None, + is_quarantined: false, + failure_message: None, + variant: None, + } + } + + fn warning(issue: JunitTestCaseValidationIssueSubOptimal) -> JunitValidationIssueType { + JunitValidationIssueType::TestCase(JunitValidationIssue::SubOptimal(issue)) + } + + fn report_warning(issue: JunitReportValidationIssueSubOptimal) -> JunitValidationIssueType { + JunitValidationIssueType::Report(JunitValidationIssue::SubOptimal(issue)) + } + + fn name_too_short() -> JunitValidationIssueType { + JunitValidationIssueType::TestCase(JunitValidationIssue::Invalid( + JunitTestCaseValidationIssueInvalid::TestCaseNameTooShort(String::new()), + )) + } + + fn validation( + all_issues: Vec, + ) -> anyhow::Result { + Ok(JunitReportValidation { + all_issues, + ..Default::default() + }) + } + + fn validations() -> JunitReportValidations { + JunitReportValidations::new(BTreeMap::from([ + ( + String::from("test-results/a.xml"), + validation(vec![ + name_too_short(), + name_too_short(), + report_warning( + JunitReportValidationIssueSubOptimal::TestCasesFileOrFilepathMissing, + ), + report_warning(JunitReportValidationIssueSubOptimal::MissingTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::FutureTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::OldTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::StaleTimestamps), + warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration), + warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseClassnameTooShort( + String::new(), + ), + ), + warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNameTooLong( + String::new(), + )), + warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseFileOrFilepathTooLong( + String::new(), + ), + ), + ]), + ), + ( + String::from("test-results/b.xml"), + Err(anyhow::anyhow!("unexpected end of file")), + ), + ( + String::from("test-results/c.xml"), + validation(vec![name_too_short()]), + ), + ( + String::from("test-results/d.xml"), + validation(vec![warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration, + )]), + ), + (String::from("test-results/e.xml"), validation(Vec::new())), + (String::from("test-results/f.xml"), validation(Vec::new())), + (String::from("test-results/g.xml"), validation(Vec::new())), + (String::from("test-results/h.xml"), validation(Vec::new())), + ( + String::from("test-results/i.xml"), + validation(vec![report_warning( + JunitReportValidationIssueSubOptimal::MissingTimestamps, + )]), + ), + ])) + } + + #[test] + fn prints_a_passing_run() { + let result = upload_result(5); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_that_no_tests_were_found() { + let result = upload_result(0); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 0 Pass: 0 Fail: 0 Quarantined: 0 Pass Ratio: N/A", + "", + "⚠️ No tests were found in the provided test results", + ] + ); + } + + #[test] + fn prints_failures_that_were_all_quarantined() { + let mut result = upload_result(4); + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .group_is_quarantined = true; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![ + failure("adds", "MathTests", Some("src/math.rs")), + failure("subtracts", "MathTests", Some("src/math.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 4 Pass: 2 Fail: 0 Quarantined: 2 Pass Ratio: 50.0%", + "", + "β€οΈβ€πŸ©Ή 2 tests failed and were quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success (0)", + ] + ); + } + + #[test] + fn prints_failures_that_were_not_quarantined() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![failure("adds", "MathTests", Some("src/math.rs"))]; + result.quarantine_context.failures = vec![ + failure("subtracts", "MathTests", Some("src/math.rs")), + failure("divides", "MathTests", Some("src/math.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 2 Fail: 2 Quarantined: 1 Pass Ratio: 40.0%", + "", + "β€οΈβ€πŸ©Ή 1 test failed and was quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "❌ 2 tests failed and were not quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/divides", + " β€· https://app.trunk.io/test-org/flaky-tests/test/divides-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn prints_a_warning_when_the_quarantine_lookup_failed() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.quarantine_context.fetch_status = + QuarantineFetchStatus::FetchFailed(anyhow::anyhow!("request timed out")); + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", + " We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn stops_at_an_error_that_blocks_quarantining() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + unauthorized_error(), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_summary(&result), + [ + "", + "Error Encountered", + "There was an unexpected error that occurred while uploading test results", + "", + "Unathorized access, your Trunk organization URL slug or token may be incorrect", + " Hint: You can find it under the settings page at https://app.trunk.io/test-org/settings", + ] + ); + } + + #[test] + fn proceeds_past_an_upload_error() { + let mut result = upload_result(3); + result.meta.base_props.bundle_upload_id = String::new(); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + anyhow::anyhow!("HTTP status client error (400 Bad Request)") + .context(ClientErrorExplanation { + code: Some(String::from("SOME_CODE")), + message: Some(String::from("Pass --test-collection-id.")), + }) + .context("Error in create bundle upload endpoint"), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_summary(&result), + [ + "", + "Error Encountered", + "There was an unexpected error that occurred while uploading test results", + "", + "Pass --test-collection-id.", + "", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + "Proceeding with test report...", + "", + "πŸ“š Test Report", + " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn prints_a_limited_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "❕ 9 files found, 5 with issues", + "", + " ❌ test-results/a.xml Has Errors And Warnings", + " β†ͺ ❌ Errors", + " β†ͺ test case name too short", + " β†ͺ test case name too short", + " β†ͺ ⚠️ Warnings", + " β†ͺ report has test cases with missing file or filepath", + " β†ͺ report has test cases with missing timestamp", + " β†ͺ report has test cases with future timestamp", + " β†ͺ report has old (> 24 hour(s)) timestamps", + " β†ͺ report has stale (> 1 hour(s)) timestamps", + " β†ͺ test case or parent has no time duration", + " β†ͺ test case classname too short", + " β†ͺ test case name too long, truncated to 1000", + " ❌ test-results/c.xml Has Errors", + " β†ͺ test case name too short", + " ⚠️ test-results/d.xml Has Warnings", + " β†ͺ test case or parent has no time duration", + "…and 1 more", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_a_full_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + result.validation_report = ValidationReport::Full; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "❕ 9 files found, 5 with issues", + "", + " ❌ test-results/a.xml Has Errors And Warnings", + " β†ͺ ❌ Errors", + " β†ͺ test case name too short", + " β†ͺ test case name too short", + " β†ͺ ⚠️ Warnings", + " β†ͺ report has test cases with missing file or filepath", + " β†ͺ report has test cases with missing timestamp", + " β†ͺ report has test cases with future timestamp", + " β†ͺ report has old (> 24 hour(s)) timestamps", + " β†ͺ report has stale (> 1 hour(s)) timestamps", + " β†ͺ test case or parent has no time duration", + " β†ͺ test case classname too short", + " β†ͺ test case name too long, truncated to 1000", + " β†ͺ test case file or filepath too long", + " ❌ test-results/c.xml Has Errors", + " β†ͺ test case name too short", + " ⚠️ test-results/d.xml Has Warnings", + " β†ͺ test case or parent has no time duration", + " ⚠️ test-results/i.xml Has Warnings", + " β†ͺ report has test cases with missing timestamp", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_a_blank_line_for_no_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + result.validation_report = ValidationReport::None; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_validation_when_every_file_is_correct() { + let mut result = upload_result(5); + result.validations = JunitReportValidations::new(BTreeMap::from([( + String::from("test-results/a.xml"), + validation(Vec::new()), + )])); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "βœ… 1 file found, all fully correct", + "", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_long_failure_messages_when_asked() { + let mut result = upload_result(1); + result.show_failure_messages = true; + result.quarantine_context.exit_code = 1; + let mut message_lines = vec![ + String::from("assertion `left == right` failed\r"), + String::from("\tleft: 1"), + String::new(), + String::from(" "), + String::from("\tright: 2"), + ]; + message_lines.extend((6..=23).map(|n| format!("frame {n}"))); + result.quarantine_context.failures = vec![Test { + failure_message: Some(message_lines.join("\n")), + ..failure("adds", "MathTests", None) + }]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 1 Pass: 0 Fail: 1 Quarantined: 0 Pass Ratio: 0.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " Failure: ", + " assertion `left == right` failed", + " left: 1", + " right: 2", + " frame 6", + " frame 7", + " frame 8", + " frame 9", + " frame 10", + " frame 11", + " frame 12", + " frame 13", + " frame 14", + " frame 15", + " frame 16", + " frame 17", + " frame 18", + " frame 19", + " frame 20", + " …and 3 more lines not shown", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn prints_the_upload_as_a_collection_link() { + let mut result = in_collection(upload_result(2)); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![Test { + id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), + ..failure("adds", "MathTests", None) + }]; + result.guid_scope = Some(TestCaseGuidScope { + test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), + repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), + }); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/tests/81d0929e-fc06-8b61-8506-991d8ebc51ea", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn links_tests_by_short_link_without_collection_ids() { + let mut result = in_collection(upload_result(2)); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn prints_the_bundle_upload_id_when_collection_links_are_hidden() { + let mut result = in_collection(upload_result(2)); + result.hide_test_collection_links = true; + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![Test { + id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), + ..failure("adds", "MathTests", None) + }]; + result.guid_scope = Some(TestCaseGuidScope { + test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), + repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), + }); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/c33a7f64-8f3e-5db9-b37b-2ea870d2441b?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn omits_the_upload_line_without_a_bundle_upload_id() { + let mut result = upload_result(5); + result.meta.base_props.bundle_upload_id = String::new(); + + assert_eq!( + upload_summary(&result), + [ + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn groups_failures_by_file_then_suite_then_neither() { + let mut result = upload_result(10); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![ + failure("parses_dates", "DateTests", Some("src/date.rs")), + failure("adds", "MathTests", None), + failure("loads", "", None), + failure("parses_times", "TimeTests", Some("src/date.rs")), + failure("subtracts", "MathTests", None), + failure("boots", "", None), + failure("formats", "", Some("src/a.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 10 Pass: 3 Fail: 7 Quarantined: 0 Pass Ratio: 30.0%", + "", + "❌ 7 tests failed and were not quarantined", + "", + " πŸ“ src/a.rs", + " formats", + " β€· https://app.trunk.io/test-org/flaky-tests/test/formats-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ“ src/date.rs", + " DateTests/parses_dates", + " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_dates-id?repo=trunk-io%2Fanalytics-cli", + " TimeTests/parses_times", + " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_times-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ” Other", + " loads", + " β€· https://app.trunk.io/test-org/flaky-tests/test/loads-id?repo=trunk-io%2Fanalytics-cli", + " boots", + " β€· https://app.trunk.io/test-org/flaky-tests/test/boots-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn shows_three_failures_per_group() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = + ["adds", "subtracts", "multiplies", "divides", "rounds"] + .into_iter() + .map(|name| failure(name, "MathTests", Some("src/math.rs"))) + .collect(); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 0 Fail: 5 Quarantined: 0 Pass Ratio: 0.0%", + "", + "❌ 5 tests failed and were not quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/multiplies", + " β€· https://app.trunk.io/test-org/flaky-tests/test/multiplies-id?repo=trunk-io%2Fanalytics-cli", + " …and 2 more failures in this group", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn warns_about_a_non_zero_exit_code_without_failures() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "⚠️ No test failures found, but non zero exit code provided: 1", + "This may indicate that some tests were not run or that there were other issues during the test run.", + ] + ); + } const URL: &str = "https://app.trunk.io/my-org/flaky-tests/test/1"; diff --git a/cli/src/validate_command.rs b/cli/src/validate_command.rs index 0b8ad1d3..27d16322 100644 --- a/cli/src/validate_command.rs +++ b/cli/src/validate_command.rs @@ -959,13 +959,34 @@ impl JunitReportValidations { #[cfg(test)] mod tests { use chrono::DateTime; - use context::junit::junit_path::{ - JunitReportFileWithTestRunnerReport, TestRunnerReport, TestRunnerReportStatus, + use context::junit::{ + junit_path::{ + JunitReportFileWithTestRunnerReport, TestRunnerReport, TestRunnerReportStatus, + }, + parser::{JunitParseIssueInvalid, JunitParseIssueSubOptimal}, }; + use pretty_assertions::assert_eq; use test_utils::inputs::get_test_file_path; use super::*; + fn validate_summary(result: &ValidateRunResult) -> Vec { + result + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() + } + + fn flat_issue(level: JunitValidationLevel, message: &str) -> JunitReportValidationFlatIssue { + JunitReportValidationFlatIssue { + level, + error_message: String::from(message), + ..Default::default() + } + } + #[test] fn test_flatten_glob_returns_all_matches() { let path = get_test_file_path("test_fixtures/junit0*"); @@ -1134,4 +1155,168 @@ mod tests { )]) ); } + + #[test] + fn prints_validation_errors_warnings_and_codeowners() { + let result = ValidateRunResult { + bep_result: None, + file_sets: vec![FileSetResult { + glob: String::from("test-results/**/*.xml"), + file_paths: ["broken", "empty", "multiple", "ok", "unit"] + .map(|name| format!("test-results/{name}.xml")) + .to_vec(), + }], + file_parse_issues: vec![ + ParseIssues { + file_path: String::from("test-results/broken.xml"), + fatal_error: Some(anyhow::anyhow!("unexpected end of file")), + errors: Vec::new(), + warnings: Vec::new(), + }, + ParseIssues { + file_path: String::from("test-results/empty.xml"), + fatal_error: None, + errors: Vec::new(), + warnings: vec![JunitParseIssue::SubOptimal( + JunitParseIssueSubOptimal::ReportNotFound, + )], + }, + ParseIssues { + file_path: String::from("test-results/multiple.xml"), + fatal_error: None, + errors: vec![JunitParseIssue::Invalid( + JunitParseIssueInvalid::ReportMultipleFound, + )], + warnings: Vec::new(), + }, + ], + test_issues: vec![ + TestIssues { + file_path: String::from("test-results/ok.xml"), + num_test_suites: 1, + num_test_cases: 3, + errors: Vec::new(), + warnings: Vec::new(), + }, + TestIssues { + file_path: String::from("test-results/unit.xml"), + num_test_suites: 2, + num_test_cases: 5, + errors: vec![flat_issue( + JunitValidationLevel::Invalid, + "test case name too short", + )], + warnings: vec![ + flat_issue( + JunitValidationLevel::SubOptimal, + "report has test cases with missing file or filepath", + ), + flat_issue( + JunitValidationLevel::SubOptimal, + "test case or parent has no time duration", + ), + ], + }, + ], + codeowners_issues: Some(CodeownersIssues { + file_path: PathBuf::from(".github/CODEOWNERS"), + warnings: vec![String::from( + "CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", + )], + }), + }; + + assert_eq!( + validate_summary(&result), + [ + "πŸ“š Validating 5 files:", + " Files matching: test-results/**/*.xml:", + " test-results/broken.xml", + " test-results/empty.xml", + " test-results/multiple.xml", + " test-results/ok.xml", + " test-results/unit.xml", + "", + "File parse issues", + "test-results/broken.xml", + "1 errors, 0 warnings", + " ❌ Errors:", + " unexpected end of file", + "", + "test-results/empty.xml", + "0 errors, 1 warnings", + " ⚠️ Warnings:", + " no reports found", + "", + "test-results/multiple.xml", + "1 errors, 0 warnings", + " ❌ Errors:", + " multiple reports found", + "", + "Test Validation Issues", + "test-results/ok.xml", + "1 test suites, 3 test cases, 0 errors, 0 warnings", + "test-results/unit.xml", + "2 test suites, 5 test cases, 1 errors, 2 warnings", + " ❌ Errors:", + " test case name too short", + " ⚠️ Warnings:", + " report has test cases with missing file or filepath", + " test case or parent has no time duration", + "", + "Checking for codeowners file...", + " Found codeowners path: \".github/CODEOWNERS\"", + " CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", + "", + "1 valid file, 2 files with warnings, and 3 files with errors, with 5 files total (a file is double counted if it has both errors and warnings)", + "3 warnings, and 3 errors", + ] + ); + } + + #[test] + fn prints_bep_parse_errors_without_a_codeowners_file() { + let result = ValidateRunResult { + bep_result: Some(BepValidateResult { + errors: vec![String::from( + "Error mapping test status for label //app:unit_test: unknown test status 99", + )], + }), + file_sets: vec![FileSetResult { + glob: String::from("bazel-testlogs/app/unit_test/test.xml"), + file_paths: vec![String::from("bazel-testlogs/app/unit_test/test.xml")], + }], + file_parse_issues: Vec::new(), + test_issues: vec![TestIssues { + file_path: String::from("bazel-testlogs/app/unit_test/test.xml"), + num_test_suites: 1, + num_test_cases: 2, + errors: Vec::new(), + warnings: Vec::new(), + }], + codeowners_issues: None, + }; + + assert_eq!( + validate_summary(&result), + [ + "⚠️ BEP file had parse errors: [\"Error mapping test status for label //app:unit_test: unknown test status 99\"]", + "", + "πŸ“š Validating 1 files:", + " Files matching: bazel-testlogs/app/unit_test/test.xml:", + " bazel-testlogs/app/unit_test/test.xml", + "", + "File parse issues", + "Test Validation Issues", + "bazel-testlogs/app/unit_test/test.xml", + "1 test suites, 2 test cases, 0 errors, 0 warnings", + "", + "Checking for codeowners file...", + " No codeowners file found", + "", + "1 valid file, 0 files with warnings, and 0 files with errors, with 1 file total (a file is double counted if it has both errors and warnings)", + "0 warnings, and 0 errors", + ] + ); + } } From 5976f94c53d33c4c7e6670f6b3fb161a9f3aad0d Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 21 Oct 2026 23:05:43 +0000 Subject: [PATCH 02/19] test(cli): move the summary tests into cli/tests The lock-down tests move from src/ into one integration test crate, cli/tests/summary, with a file each for upload, validate and ErrorReport. The existing ErrorReport output test moves with them and now checks every line. report_limiting becomes public so the tests can set --validation-report. The validate tests now run run_validate on fixture files instead of building ValidateRunResult, whose fields are private. Refs TRUNK-22399 Co-Authored-By: Claude Opus 5.5 (1M context) --- cli/src/error_report.rs | 186 ---- cli/src/lib.rs | 2 +- cli/src/upload_command.rs | 978 ++---------------- cli/src/validate_command.rs | 189 +--- .../summary/bep/bep_with_parse_error | 2 + cli/test_fixtures/summary/bep/test.xml | 6 + cli/test_fixtures/summary/validate/CODEOWNERS | 1 + cli/test_fixtures/summary/validate/broken.xml | 4 + cli/test_fixtures/summary/validate/empty.xml | 0 .../summary/validate/multiple.xml | 11 + cli/test_fixtures/summary/validate/unit.xml | 8 + cli/tests/summary/error_report.rs | 183 ++++ cli/tests/summary/main.rs | 12 + cli/tests/summary/upload.rs | 817 +++++++++++++++ cli/tests/summary/validate.rs | 110 ++ 15 files changed, 1242 insertions(+), 1267 deletions(-) create mode 100644 cli/test_fixtures/summary/bep/bep_with_parse_error create mode 100644 cli/test_fixtures/summary/bep/test.xml create mode 100644 cli/test_fixtures/summary/validate/CODEOWNERS create mode 100644 cli/test_fixtures/summary/validate/broken.xml create mode 100644 cli/test_fixtures/summary/validate/empty.xml create mode 100644 cli/test_fixtures/summary/validate/multiple.xml create mode 100644 cli/test_fixtures/summary/validate/unit.xml create mode 100644 cli/tests/summary/error_report.rs create mode 100644 cli/tests/summary/main.rs create mode 100644 cli/tests/summary/upload.rs create mode 100644 cli/tests/summary/validate.rs diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index 98ea9b20..c13284f2 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -240,32 +240,6 @@ impl EndOutput for ErrorReport { } } -#[cfg(test)] -fn error_summary(report: &ErrorReport) -> Vec { - report - .output() - .unwrap() - .iter() - .map(Line::to_unstyled) - .collect() -} - -#[cfg(test)] -pub(crate) fn unauthorized_error() -> anyhow::Error { - let response = http::Response::builder().status(401).body("").unwrap(); - anyhow::Error::from( - reqwest::Response::from(response) - .error_for_status() - .unwrap_err(), - ) -} - -#[cfg(test)] -fn connection_refused_error() -> anyhow::Error { - anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)) - .context("Failed to create bundle upload") -} - #[test] fn adds_settings_if_domain_present() { let host = "https://api.fake-trunk.io"; @@ -275,163 +249,3 @@ fn adds_settings_if_domain_present() { "Hint: You can find it under the settings page at https://app.fake-trunk.io/fake-org-slug/settings" ) } - -#[test] -fn prints_a_client_error_explanation_beneath_the_base_message() { - let error = anyhow::anyhow!("HTTP status client error (400 Bad Request)") - .context(ClientErrorExplanation { - code: Some(String::from("SOME_CODE")), - message: Some(String::from("Pass --test-collection-id.")), - }) - .context("Error in create bundle upload endpoint"); - let report = ErrorReport::new( - error, - String::from("fake-org-slug"), - Some(String::from("There was an unexpected error")), - ); - - let text = error_summary(&report); - - assert_eq!( - text[..4], - [ - "Error Encountered", - "There was an unexpected error", - "", - "Pass --test-collection-id.", - ] - ); -} - -#[test] -fn prints_connection_refused_beneath_the_base_message() { - let report = ErrorReport::new( - connection_refused_error(), - String::from("fake-org-slug"), - Some(String::from("There was an unexpected error")), - ); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "There was an unexpected error", - "", - "Unable to connect to trunk's server", - ] - ); -} - -#[test] -fn prints_connection_refused_without_a_base_message() { - let report = ErrorReport::new( - connection_refused_error(), - String::from("fake-org-slug"), - None, - ); - - assert_eq!( - error_summary(&report), - ["Error Encountered", "Unable to connect to trunk's server"] - ); -} - -#[test] -fn prints_unauthorized_with_a_settings_hint() { - let report = ErrorReport::new(unauthorized_error(), String::from("fake-org-slug"), None); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "", - "", - "Unathorized access, your Trunk organization URL slug or token may be incorrect", - " Hint: You can find it under the settings page at https://app.trunk.io/fake-org-slug/settings", - ] - ); -} - -#[test] -fn prints_a_generic_error_without_a_base_message() { - let report = ErrorReport::new( - anyhow::anyhow!("--allow-forked-pr-uploads requires --test-collection-id"), - String::from("fake-org-slug"), - None, - ); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "--allow-forked-pr-uploads requires --test-collection-id", - "For more help, contact us at https://slack.trunk.io/", - "", - "Errors occurred during execution, using quarantining exit code", - ] - ); -} - -#[test] -fn prints_an_interrupting_error_with_its_exit_code() { - let report = ErrorReport::new( - anyhow::anyhow!(InterruptingError::new( - "No test output files found to validate" - )), - String::from("fake-org-slug"), - None, - ); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "No test output files found to validate", - "For more help, contact us at https://slack.trunk.io/", - "", - "Errors occurred during execution, returning exit code: 70", - ] - ); -} - -#[test] -fn prints_the_default_exit_code() { - let mut report = ErrorReport::new( - anyhow::anyhow!("something went wrong"), - String::from("fake-org-slug"), - None, - ); - report.context.exit_code = Some(exitcode::OK); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "something went wrong", - "For more help, contact us at https://slack.trunk.io/", - "", - "No errors occurred, returning default exit code: 0", - ] - ); -} - -#[test] -fn prints_any_other_exit_code() { - let mut report = ErrorReport::new( - anyhow::anyhow!("something went wrong"), - String::from("fake-org-slug"), - None, - ); - report.context.exit_code = Some(exitcode::USAGE); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "something went wrong", - "For more help, contact us at https://slack.trunk.io/", - "", - "Errors occurred during execution, returning exit code: 64", - ] - ); -} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 0c7bd3ba..92ce0bd4 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -2,7 +2,7 @@ pub mod context; pub mod context_quarantine; pub mod error_report; pub mod print; -mod report_limiting; +pub mod report_limiting; pub mod test_command; pub mod upload_command; pub mod validate_command; diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index 9e35fc2b..34425fc4 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -1080,94 +1080,96 @@ impl EndOutput for UploadRunResult { } } // Helper to render a group - let mut render_group = - |header: &str, color: Color, group: &Vec<&Test>| -> anyhow::Result<()> { - output.push(Line::from_iter([Span::new_styled( - style(header.to_string()) - .with(color) - .attribute(Attribute::Bold), - )?])); - for test in group.iter().take(3) { - let output_name = format!( - "{}{}{}", - test.parent_name, - if test.parent_name.is_empty() { "" } else { "/" }, - test.name - ); - let mut test_line = Line::from_iter([Span::new_styled_lossy( - style(output_name.to_string()).attribute(Attribute::Bold), - )]); - test_line.pad_left(2); - output.push(test_line); - let link = url_for_test_case( - &self.api_address, - &self.quarantine_context.org_url_slug, - &self.quarantine_context.repo, - test, - test_collection_short_id, - guid_scope, - )?; - let mut link_output = Line::from_iter([ - Span::new_unstyled("β€· ")?, - hyperlink_span(link, Attribute::Underlined)?, - ]); - link_output.pad_left(4); - output.push(link_output); - // Display failure message if present and enabled - // TODO: show_failure_messages is a temporary flag to show failure messages - // in the output. It should be removed once we are confident in this flow - // and we should use the validation report flag instead. - if self.show_failure_messages && test.failure_message.is_some() { - let failure_message = test.failure_message.as_ref().unwrap(); - let lines: Vec<&str> = failure_message.split('\n').collect(); - let max_lines = 20; - let shown_lines = lines.iter().take(max_lines); - let mut failure_header = Line::from_iter([Span::new_styled( - style("Failure: ".to_string()).with(Color::DarkGrey), - )?]); - failure_header.pad_left(4); - output.push(failure_header); - for (j, line) in shown_lines.enumerate() { - let sanitized_line = line.replace('\t', " ").replace('\r', ""); - if !sanitized_line.trim().is_empty() || j == 0 { - let mut failure_output = Line::from_iter([ - Span::new_unstyled(" ")?, - Span::new_styled_lossy( - style(sanitized_line.to_string()) - .with(Color::Grey) - .attribute(Attribute::Italic), - ), - ]); - failure_output.pad_left(4); - output.push(failure_output); - } - } - if lines.len() > max_lines { - let omitted = lines.len() - max_lines; - let mut more_output = Line::from_iter([ + let mut render_group = |header: &str, + color: Color, + group: &Vec<&Test>| + -> anyhow::Result<()> { + output.push(Line::from_iter([Span::new_styled( + style(header.to_string()) + .with(color) + .attribute(Attribute::Bold), + )?])); + for test in group.iter().take(3) { + let output_name = format!( + "{}{}{}", + test.parent_name, + if test.parent_name.is_empty() { "" } else { "/" }, + test.name + ); + let mut test_line = Line::from_iter([Span::new_styled_lossy( + style(output_name.to_string()).attribute(Attribute::Bold), + )]); + test_line.pad_left(2); + output.push(test_line); + let link = url_for_test_case( + &self.api_address, + &self.quarantine_context.org_url_slug, + &self.quarantine_context.repo, + test, + test_collection_short_id, + guid_scope, + )?; + let mut link_output = Line::from_iter([ + Span::new_unstyled("β€· ")?, + hyperlink_span(link, Attribute::Underlined)?, + ]); + link_output.pad_left(4); + output.push(link_output); + // Display failure message if present and enabled + // TODO: show_failure_messages is a temporary flag to show failure messages + // in the output. It should be removed once we are confident in this flow + // and we should use the validation report flag instead. + if self.show_failure_messages && test.failure_message.is_some() { + let failure_message = test.failure_message.as_ref().unwrap(); + let lines: Vec<&str> = failure_message.split('\n').collect(); + let max_lines = 20; + let shown_lines = lines.iter().take(max_lines); + let mut failure_header = Line::from_iter([Span::new_styled( + style("Failure: ".to_string()).with(Color::DarkGrey), + )?]); + failure_header.pad_left(4); + output.push(failure_header); + for (j, line) in shown_lines.enumerate() { + let sanitized_line = line.replace('\t', " ").replace('\r', ""); + if !sanitized_line.trim().is_empty() || j == 0 { + let mut failure_output = Line::from_iter([ Span::new_unstyled(" ")?, - Span::new_styled( - style(format!("…and {omitted} more lines not shown")) + Span::new_styled_lossy( + style(sanitized_line.to_string()) .with(Color::Grey) .attribute(Attribute::Italic), - )?, + ), ]); - more_output.pad_left(4); - output.push(more_output); + failure_output.pad_left(4); + output.push(failure_output); } } + if lines.len() > max_lines { + let omitted = lines.len() - max_lines; + let mut more_output = Line::from_iter([ + Span::new_unstyled(" ")?, + Span::new_styled( + style(format!("…and {omitted} more lines not shown")) + .with(Color::Grey) + .attribute(Attribute::Italic), + )?, + ]); + more_output.pad_left(4); + output.push(more_output); + } } - if group.len() > 3 { - let mut more_failures = Line::from_iter([Span::new_unstyled(format!( - "…and {} more failures in this group", - group.len() - 3 - ))?]); - more_failures.pad_left(2); - output.push(more_failures); - } - output.push(Line::default()); - Ok(()) - }; + } + if group.len() > 3 { + let mut more_failures = Line::from_iter([Span::new_unstyled(format!( + "…and {} more failures in this group", + group.len() - 3 + ))?]); + more_failures.pad_left(2); + output.push(more_failures); + } + output.push(Line::default()); + Ok(()) + }; // Render file path groups (cyan) for (file, group) in &groups { render_group(&format!("πŸ“ {}", file), Color::Cyan, group)?; @@ -1288,819 +1290,9 @@ impl EndOutput for UploadRunResult { #[cfg(test)] mod tests { - use std::collections::{BTreeMap, HashMap}; - - use api::{client::ClientErrorExplanation, urls::TestCaseGuidScope}; - use bundle::{ - BundleMetaBaseProps, BundleMetaDebugProps, BundleMetaJunitProps, TestCollectionProps, - }; - use context::{ - junit::validator::{ - JunitReportValidation, JunitReportValidationIssueSubOptimal, - JunitTestCaseValidationIssueInvalid, JunitTestCaseValidationIssueSubOptimal, - JunitValidationIssue, JunitValidationIssueType, - }, - repo::{BundleRepo, RepoUrlParts}, - }; - use pretty_assertions::assert_eq; - - use super::*; - use crate::{context_quarantine::QuarantineFetchStatus, error_report::unauthorized_error}; - - const BUNDLE_UPLOAD_ID: &str = "4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d"; - const UPLOAD_ERROR_MESSAGE: &str = - "There was an unexpected error that occurred while uploading test results"; - - fn upload_summary(result: &UploadRunResult) -> Vec { - result - .output() - .unwrap() - .iter() - .map(Line::to_unstyled) - .collect() - } - - fn repo() -> RepoUrlParts { - RepoUrlParts { - host: String::from("github.com"), - owner: String::from("trunk-io"), - name: String::from("analytics-cli"), - } - } - - fn upload_result(num_tests: usize) -> UploadRunResult { - let mut quarantine_context = QuarantineContext::skip_fetch(Vec::new()); - quarantine_context.repo = repo(); - quarantine_context.org_url_slug = String::from("test-org"); - UploadRunResult { - error_report: None, - quarantine_context, - meta: BundleMeta { - base_props: BundleMetaBaseProps { - version: String::new(), - cli_version: String::new(), - org: String::from("test-org"), - test_collection: None, - repo: BundleRepo { - repo: repo(), - ..Default::default() - }, - bundle_upload_id: String::from(BUNDLE_UPLOAD_ID), - tags: Vec::new(), - file_sets: Vec::new(), - envs: HashMap::new(), - upload_time_epoch: 0, - test_command: None, - os_info: None, - quarantined_tests: Vec::new(), - codeowners: None, - use_uncloned_repo: None, - quarantine_resolution_mode: QuarantineResolutionMode::default(), - }, - junit_props: BundleMetaJunitProps { - num_files: 1, - num_tests, - }, - debug_props: BundleMetaDebugProps { - command_line: String::new(), - trunk_envs: HashMap::new(), - }, - bundle_upload_id_v2: String::new(), - variant: None, - internal_bundled_file: None, - failed_tests: Vec::new(), - }, - validations: JunitReportValidations::new(BTreeMap::new()), - validation_report: ValidationReport::Limited, - show_failure_messages: false, - test_collection_short_id: None, - hide_test_collection_links: false, - api_address: String::from("https://api.trunk.io"), - guid_scope: None, - } - } - - fn in_collection(mut result: UploadRunResult) -> UploadRunResult { - result.test_collection_short_id = Some(String::from("abc123")); - result.meta.base_props.test_collection = Some(TestCollectionProps { - short_id: String::from("abc123"), - bundle_meta_id: String::from("0b6f4c1e-8a7d-4e2b-9c3f-5d1a2b3c4d5e"), - bundle_meta_created_at: String::from("2026-10-21T12:00:00Z"), - }); - result - } - - fn failure(name: &str, parent_name: &str, file: Option<&str>) -> Test { - Test { - name: String::from(name), - parent_name: String::from(parent_name), - class_name: None, - file: file.map(String::from), - id: format!("{name}-id"), - timestamp_millis: None, - is_quarantined: false, - failure_message: None, - variant: None, - } - } - - fn warning(issue: JunitTestCaseValidationIssueSubOptimal) -> JunitValidationIssueType { - JunitValidationIssueType::TestCase(JunitValidationIssue::SubOptimal(issue)) - } - - fn report_warning(issue: JunitReportValidationIssueSubOptimal) -> JunitValidationIssueType { - JunitValidationIssueType::Report(JunitValidationIssue::SubOptimal(issue)) - } - - fn name_too_short() -> JunitValidationIssueType { - JunitValidationIssueType::TestCase(JunitValidationIssue::Invalid( - JunitTestCaseValidationIssueInvalid::TestCaseNameTooShort(String::new()), - )) - } - - fn validation( - all_issues: Vec, - ) -> anyhow::Result { - Ok(JunitReportValidation { - all_issues, - ..Default::default() - }) - } - - fn validations() -> JunitReportValidations { - JunitReportValidations::new(BTreeMap::from([ - ( - String::from("test-results/a.xml"), - validation(vec![ - name_too_short(), - name_too_short(), - report_warning( - JunitReportValidationIssueSubOptimal::TestCasesFileOrFilepathMissing, - ), - report_warning(JunitReportValidationIssueSubOptimal::MissingTimestamps), - report_warning(JunitReportValidationIssueSubOptimal::FutureTimestamps), - report_warning(JunitReportValidationIssueSubOptimal::OldTimestamps), - report_warning(JunitReportValidationIssueSubOptimal::StaleTimestamps), - warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration), - warning( - JunitTestCaseValidationIssueSubOptimal::TestCaseClassnameTooShort( - String::new(), - ), - ), - warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNameTooLong( - String::new(), - )), - warning( - JunitTestCaseValidationIssueSubOptimal::TestCaseFileOrFilepathTooLong( - String::new(), - ), - ), - ]), - ), - ( - String::from("test-results/b.xml"), - Err(anyhow::anyhow!("unexpected end of file")), - ), - ( - String::from("test-results/c.xml"), - validation(vec![name_too_short()]), - ), - ( - String::from("test-results/d.xml"), - validation(vec![warning( - JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration, - )]), - ), - (String::from("test-results/e.xml"), validation(Vec::new())), - (String::from("test-results/f.xml"), validation(Vec::new())), - (String::from("test-results/g.xml"), validation(Vec::new())), - (String::from("test-results/h.xml"), validation(Vec::new())), - ( - String::from("test-results/i.xml"), - validation(vec![report_warning( - JunitReportValidationIssueSubOptimal::MissingTimestamps, - )]), - ), - ])) - } - - #[test] - fn prints_a_passing_run() { - let result = upload_result(5); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_that_no_tests_were_found() { - let result = upload_result(0); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 0 Pass: 0 Fail: 0 Quarantined: 0 Pass Ratio: N/A", - "", - "⚠️ No tests were found in the provided test results", - ] - ); - } + use superconsole::{Line, style::Attribute}; - #[test] - fn prints_failures_that_were_all_quarantined() { - let mut result = upload_result(4); - result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; - result - .quarantine_context - .quarantine_status - .group_is_quarantined = true; - result - .quarantine_context - .quarantine_status - .quarantine_results = vec![ - failure("adds", "MathTests", Some("src/math.rs")), - failure("subtracts", "MathTests", Some("src/math.rs")), - ]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 4 Pass: 2 Fail: 0 Quarantined: 2 Pass Ratio: 50.0%", - "", - "β€οΈβ€πŸ©Ή 2 tests failed and were quarantined", - "", - " πŸ“ src/math.rs", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/subtracts", - " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success (0)", - ] - ); - } - - #[test] - fn prints_failures_that_were_not_quarantined() { - let mut result = upload_result(5); - result.quarantine_context.exit_code = 1; - result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; - result - .quarantine_context - .quarantine_status - .quarantine_results = vec![failure("adds", "MathTests", Some("src/math.rs"))]; - result.quarantine_context.failures = vec![ - failure("subtracts", "MathTests", Some("src/math.rs")), - failure("divides", "MathTests", Some("src/math.rs")), - ]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 2 Fail: 2 Quarantined: 1 Pass Ratio: 40.0%", - "", - "β€οΈβ€πŸ©Ή 1 test failed and was quarantined", - "", - " πŸ“ src/math.rs", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "❌ 2 tests failed and were not quarantined", - "", - " πŸ“ src/math.rs", - " MathTests/subtracts", - " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/divides", - " β€· https://app.trunk.io/test-org/flaky-tests/test/divides-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn prints_a_warning_when_the_quarantine_lookup_failed() { - let mut result = upload_result(3); - result.quarantine_context.exit_code = 1; - result.quarantine_context.fetch_status = - QuarantineFetchStatus::FetchFailed(anyhow::anyhow!("request timed out")); - result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", - " We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn stops_at_an_error_that_blocks_quarantining() { - let mut result = upload_result(3); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; - result.error_report = Some(ErrorReport::new( - unauthorized_error(), - String::from("test-org"), - Some(String::from(UPLOAD_ERROR_MESSAGE)), - )); - - assert_eq!( - upload_summary(&result), - [ - "", - "Error Encountered", - "There was an unexpected error that occurred while uploading test results", - "", - "Unathorized access, your Trunk organization URL slug or token may be incorrect", - " Hint: You can find it under the settings page at https://app.trunk.io/test-org/settings", - ] - ); - } - - #[test] - fn proceeds_past_an_upload_error() { - let mut result = upload_result(3); - result.meta.base_props.bundle_upload_id = String::new(); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; - result.error_report = Some(ErrorReport::new( - anyhow::anyhow!("HTTP status client error (400 Bad Request)") - .context(ClientErrorExplanation { - code: Some(String::from("SOME_CODE")), - message: Some(String::from("Pass --test-collection-id.")), - }) - .context("Error in create bundle upload endpoint"), - String::from("test-org"), - Some(String::from(UPLOAD_ERROR_MESSAGE)), - )); - - assert_eq!( - upload_summary(&result), - [ - "", - "Error Encountered", - "There was an unexpected error that occurred while uploading test results", - "", - "Pass --test-collection-id.", - "", - "For more help, contact us at https://slack.trunk.io/", - "", - "Errors occurred during execution, using quarantining exit code", - "Proceeding with test report...", - "", - "πŸ“š Test Report", - " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn prints_a_limited_validation_report() { - let mut result = upload_result(5); - result.validations = validations(); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“‚ File Validation", - "", - "❕ 9 files found, 5 with issues", - "", - " ❌ test-results/a.xml Has Errors And Warnings", - " β†ͺ ❌ Errors", - " β†ͺ test case name too short", - " β†ͺ test case name too short", - " β†ͺ ⚠️ Warnings", - " β†ͺ report has test cases with missing file or filepath", - " β†ͺ report has test cases with missing timestamp", - " β†ͺ report has test cases with future timestamp", - " β†ͺ report has old (> 24 hour(s)) timestamps", - " β†ͺ report has stale (> 1 hour(s)) timestamps", - " β†ͺ test case or parent has no time duration", - " β†ͺ test case classname too short", - " β†ͺ test case name too long, truncated to 1000", - " ❌ test-results/c.xml Has Errors", - " β†ͺ test case name too short", - " ⚠️ test-results/d.xml Has Warnings", - " β†ͺ test case or parent has no time duration", - "…and 1 more", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_a_full_validation_report() { - let mut result = upload_result(5); - result.validations = validations(); - result.validation_report = ValidationReport::Full; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“‚ File Validation", - "", - "❕ 9 files found, 5 with issues", - "", - " ❌ test-results/a.xml Has Errors And Warnings", - " β†ͺ ❌ Errors", - " β†ͺ test case name too short", - " β†ͺ test case name too short", - " β†ͺ ⚠️ Warnings", - " β†ͺ report has test cases with missing file or filepath", - " β†ͺ report has test cases with missing timestamp", - " β†ͺ report has test cases with future timestamp", - " β†ͺ report has old (> 24 hour(s)) timestamps", - " β†ͺ report has stale (> 1 hour(s)) timestamps", - " β†ͺ test case or parent has no time duration", - " β†ͺ test case classname too short", - " β†ͺ test case name too long, truncated to 1000", - " β†ͺ test case file or filepath too long", - " ❌ test-results/c.xml Has Errors", - " β†ͺ test case name too short", - " ⚠️ test-results/d.xml Has Warnings", - " β†ͺ test case or parent has no time duration", - " ⚠️ test-results/i.xml Has Warnings", - " β†ͺ report has test cases with missing timestamp", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_a_blank_line_for_no_validation_report() { - let mut result = upload_result(5); - result.validations = validations(); - result.validation_report = ValidationReport::None; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_validation_when_every_file_is_correct() { - let mut result = upload_result(5); - result.validations = JunitReportValidations::new(BTreeMap::from([( - String::from("test-results/a.xml"), - validation(Vec::new()), - )])); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“‚ File Validation", - "", - "βœ… 1 file found, all fully correct", - "", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_long_failure_messages_when_asked() { - let mut result = upload_result(1); - result.show_failure_messages = true; - result.quarantine_context.exit_code = 1; - let mut message_lines = vec![ - String::from("assertion `left == right` failed\r"), - String::from("\tleft: 1"), - String::new(), - String::from(" "), - String::from("\tright: 2"), - ]; - message_lines.extend((6..=23).map(|n| format!("frame {n}"))); - result.quarantine_context.failures = vec![Test { - failure_message: Some(message_lines.join("\n")), - ..failure("adds", "MathTests", None) - }]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 1 Pass: 0 Fail: 1 Quarantined: 0 Pass Ratio: 0.0%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " Failure: ", - " assertion `left == right` failed", - " left: 1", - " right: 2", - " frame 6", - " frame 7", - " frame 8", - " frame 9", - " frame 10", - " frame 11", - " frame 12", - " frame 13", - " frame 14", - " frame 15", - " frame 16", - " frame 17", - " frame 18", - " frame 19", - " frame 20", - " …and 3 more lines not shown", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn prints_the_upload_as_a_collection_link() { - let mut result = in_collection(upload_result(2)); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![Test { - id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), - ..failure("adds", "MathTests", None) - }]; - result.guid_scope = Some(TestCaseGuidScope { - test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), - repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), - }); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", - "", - "πŸ“š Test Report", - " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/tests/81d0929e-fc06-8b61-8506-991d8ebc51ea", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn links_tests_by_short_link_without_collection_ids() { - let mut result = in_collection(upload_result(2)); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", - "", - "πŸ“š Test Report", - " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/adds-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn prints_the_bundle_upload_id_when_collection_links_are_hidden() { - let mut result = in_collection(upload_result(2)); - result.hide_test_collection_links = true; - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![Test { - id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), - ..failure("adds", "MathTests", None) - }]; - result.guid_scope = Some(TestCaseGuidScope { - test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), - repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), - }); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/c33a7f64-8f3e-5db9-b37b-2ea870d2441b?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn omits_the_upload_line_without_a_bundle_upload_id() { - let mut result = upload_result(5); - result.meta.base_props.bundle_upload_id = String::new(); - - assert_eq!( - upload_summary(&result), - [ - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn groups_failures_by_file_then_suite_then_neither() { - let mut result = upload_result(10); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![ - failure("parses_dates", "DateTests", Some("src/date.rs")), - failure("adds", "MathTests", None), - failure("loads", "", None), - failure("parses_times", "TimeTests", Some("src/date.rs")), - failure("subtracts", "MathTests", None), - failure("boots", "", None), - failure("formats", "", Some("src/a.rs")), - ]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 10 Pass: 3 Fail: 7 Quarantined: 0 Pass Ratio: 30.0%", - "", - "❌ 7 tests failed and were not quarantined", - "", - " πŸ“ src/a.rs", - " formats", - " β€· https://app.trunk.io/test-org/flaky-tests/test/formats-id?repo=trunk-io%2Fanalytics-cli", - " ", - " πŸ“ src/date.rs", - " DateTests/parses_dates", - " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_dates-id?repo=trunk-io%2Fanalytics-cli", - " TimeTests/parses_times", - " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_times-id?repo=trunk-io%2Fanalytics-cli", - " ", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/subtracts", - " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", - " ", - " πŸ” Other", - " loads", - " β€· https://app.trunk.io/test-org/flaky-tests/test/loads-id?repo=trunk-io%2Fanalytics-cli", - " boots", - " β€· https://app.trunk.io/test-org/flaky-tests/test/boots-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn shows_three_failures_per_group() { - let mut result = upload_result(5); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = - ["adds", "subtracts", "multiplies", "divides", "rounds"] - .into_iter() - .map(|name| failure(name, "MathTests", Some("src/math.rs"))) - .collect(); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 0 Fail: 5 Quarantined: 0 Pass Ratio: 0.0%", - "", - "❌ 5 tests failed and were not quarantined", - "", - " πŸ“ src/math.rs", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/subtracts", - " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/multiplies", - " β€· https://app.trunk.io/test-org/flaky-tests/test/multiplies-id?repo=trunk-io%2Fanalytics-cli", - " …and 2 more failures in this group", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn warns_about_a_non_zero_exit_code_without_failures() { - let mut result = upload_result(5); - result.quarantine_context.exit_code = 1; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "⚠️ No test failures found, but non zero exit code provided: 1", - "This may indicate that some tests were not run or that there were other issues during the test run.", - ] - ); - } + use super::url_span; const URL: &str = "https://app.trunk.io/my-org/flaky-tests/test/1"; diff --git a/cli/src/validate_command.rs b/cli/src/validate_command.rs index 27d16322..0b8ad1d3 100644 --- a/cli/src/validate_command.rs +++ b/cli/src/validate_command.rs @@ -959,34 +959,13 @@ impl JunitReportValidations { #[cfg(test)] mod tests { use chrono::DateTime; - use context::junit::{ - junit_path::{ - JunitReportFileWithTestRunnerReport, TestRunnerReport, TestRunnerReportStatus, - }, - parser::{JunitParseIssueInvalid, JunitParseIssueSubOptimal}, + use context::junit::junit_path::{ + JunitReportFileWithTestRunnerReport, TestRunnerReport, TestRunnerReportStatus, }; - use pretty_assertions::assert_eq; use test_utils::inputs::get_test_file_path; use super::*; - fn validate_summary(result: &ValidateRunResult) -> Vec { - result - .output() - .unwrap() - .iter() - .map(Line::to_unstyled) - .collect() - } - - fn flat_issue(level: JunitValidationLevel, message: &str) -> JunitReportValidationFlatIssue { - JunitReportValidationFlatIssue { - level, - error_message: String::from(message), - ..Default::default() - } - } - #[test] fn test_flatten_glob_returns_all_matches() { let path = get_test_file_path("test_fixtures/junit0*"); @@ -1155,168 +1134,4 @@ mod tests { )]) ); } - - #[test] - fn prints_validation_errors_warnings_and_codeowners() { - let result = ValidateRunResult { - bep_result: None, - file_sets: vec![FileSetResult { - glob: String::from("test-results/**/*.xml"), - file_paths: ["broken", "empty", "multiple", "ok", "unit"] - .map(|name| format!("test-results/{name}.xml")) - .to_vec(), - }], - file_parse_issues: vec![ - ParseIssues { - file_path: String::from("test-results/broken.xml"), - fatal_error: Some(anyhow::anyhow!("unexpected end of file")), - errors: Vec::new(), - warnings: Vec::new(), - }, - ParseIssues { - file_path: String::from("test-results/empty.xml"), - fatal_error: None, - errors: Vec::new(), - warnings: vec![JunitParseIssue::SubOptimal( - JunitParseIssueSubOptimal::ReportNotFound, - )], - }, - ParseIssues { - file_path: String::from("test-results/multiple.xml"), - fatal_error: None, - errors: vec![JunitParseIssue::Invalid( - JunitParseIssueInvalid::ReportMultipleFound, - )], - warnings: Vec::new(), - }, - ], - test_issues: vec![ - TestIssues { - file_path: String::from("test-results/ok.xml"), - num_test_suites: 1, - num_test_cases: 3, - errors: Vec::new(), - warnings: Vec::new(), - }, - TestIssues { - file_path: String::from("test-results/unit.xml"), - num_test_suites: 2, - num_test_cases: 5, - errors: vec![flat_issue( - JunitValidationLevel::Invalid, - "test case name too short", - )], - warnings: vec![ - flat_issue( - JunitValidationLevel::SubOptimal, - "report has test cases with missing file or filepath", - ), - flat_issue( - JunitValidationLevel::SubOptimal, - "test case or parent has no time duration", - ), - ], - }, - ], - codeowners_issues: Some(CodeownersIssues { - file_path: PathBuf::from(".github/CODEOWNERS"), - warnings: vec![String::from( - "CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", - )], - }), - }; - - assert_eq!( - validate_summary(&result), - [ - "πŸ“š Validating 5 files:", - " Files matching: test-results/**/*.xml:", - " test-results/broken.xml", - " test-results/empty.xml", - " test-results/multiple.xml", - " test-results/ok.xml", - " test-results/unit.xml", - "", - "File parse issues", - "test-results/broken.xml", - "1 errors, 0 warnings", - " ❌ Errors:", - " unexpected end of file", - "", - "test-results/empty.xml", - "0 errors, 1 warnings", - " ⚠️ Warnings:", - " no reports found", - "", - "test-results/multiple.xml", - "1 errors, 0 warnings", - " ❌ Errors:", - " multiple reports found", - "", - "Test Validation Issues", - "test-results/ok.xml", - "1 test suites, 3 test cases, 0 errors, 0 warnings", - "test-results/unit.xml", - "2 test suites, 5 test cases, 1 errors, 2 warnings", - " ❌ Errors:", - " test case name too short", - " ⚠️ Warnings:", - " report has test cases with missing file or filepath", - " test case or parent has no time duration", - "", - "Checking for codeowners file...", - " Found codeowners path: \".github/CODEOWNERS\"", - " CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", - "", - "1 valid file, 2 files with warnings, and 3 files with errors, with 5 files total (a file is double counted if it has both errors and warnings)", - "3 warnings, and 3 errors", - ] - ); - } - - #[test] - fn prints_bep_parse_errors_without_a_codeowners_file() { - let result = ValidateRunResult { - bep_result: Some(BepValidateResult { - errors: vec![String::from( - "Error mapping test status for label //app:unit_test: unknown test status 99", - )], - }), - file_sets: vec![FileSetResult { - glob: String::from("bazel-testlogs/app/unit_test/test.xml"), - file_paths: vec![String::from("bazel-testlogs/app/unit_test/test.xml")], - }], - file_parse_issues: Vec::new(), - test_issues: vec![TestIssues { - file_path: String::from("bazel-testlogs/app/unit_test/test.xml"), - num_test_suites: 1, - num_test_cases: 2, - errors: Vec::new(), - warnings: Vec::new(), - }], - codeowners_issues: None, - }; - - assert_eq!( - validate_summary(&result), - [ - "⚠️ BEP file had parse errors: [\"Error mapping test status for label //app:unit_test: unknown test status 99\"]", - "", - "πŸ“š Validating 1 files:", - " Files matching: bazel-testlogs/app/unit_test/test.xml:", - " bazel-testlogs/app/unit_test/test.xml", - "", - "File parse issues", - "Test Validation Issues", - "bazel-testlogs/app/unit_test/test.xml", - "1 test suites, 2 test cases, 0 errors, 0 warnings", - "", - "Checking for codeowners file...", - " No codeowners file found", - "", - "1 valid file, 0 files with warnings, and 0 files with errors, with 1 file total (a file is double counted if it has both errors and warnings)", - "0 warnings, and 0 errors", - ] - ); - } } diff --git a/cli/test_fixtures/summary/bep/bep_with_parse_error b/cli/test_fixtures/summary/bep/bep_with_parse_error new file mode 100644 index 00000000..f50844ef --- /dev/null +++ b/cli/test_fixtures/summary/bep/bep_with_parse_error @@ -0,0 +1,2 @@ +{"id":{"testResult":{"label":"//app:unit_test","run":1,"shard":1,"attempt":1}},"testResult":{"testActionOutput":[{"name":"test.xml","uri":"file://test_fixtures/summary/bep/test.xml"}],"status":"PASSED"}} +{"id":{"testResult": diff --git a/cli/test_fixtures/summary/bep/test.xml b/cli/test_fixtures/summary/bep/test.xml new file mode 100644 index 00000000..f9f77a91 --- /dev/null +++ b/cli/test_fixtures/summary/bep/test.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/cli/test_fixtures/summary/validate/CODEOWNERS b/cli/test_fixtures/summary/validate/CODEOWNERS new file mode 100644 index 00000000..b1ece579 --- /dev/null +++ b/cli/test_fixtures/summary/validate/CODEOWNERS @@ -0,0 +1 @@ +* @example-org/example-team diff --git a/cli/test_fixtures/summary/validate/broken.xml b/cli/test_fixtures/summary/validate/broken.xml new file mode 100644 index 00000000..a855a14f --- /dev/null +++ b/cli/test_fixtures/summary/validate/broken.xml @@ -0,0 +1,4 @@ + + + + diff --git a/cli/test_fixtures/summary/validate/empty.xml b/cli/test_fixtures/summary/validate/empty.xml new file mode 100644 index 00000000..e69de29b diff --git a/cli/test_fixtures/summary/validate/multiple.xml b/cli/test_fixtures/summary/validate/multiple.xml new file mode 100644 index 00000000..314d88a3 --- /dev/null +++ b/cli/test_fixtures/summary/validate/multiple.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/cli/test_fixtures/summary/validate/unit.xml b/cli/test_fixtures/summary/validate/unit.xml new file mode 100644 index 00000000..76812c04 --- /dev/null +++ b/cli/test_fixtures/summary/validate/unit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/cli/tests/summary/error_report.rs b/cli/tests/summary/error_report.rs new file mode 100644 index 00000000..d946938b --- /dev/null +++ b/cli/tests/summary/error_report.rs @@ -0,0 +1,183 @@ +use api::client::ClientErrorExplanation; +use display::end_output::EndOutput; +use pretty_assertions::assert_eq; +use superconsole::Line; +use trunk_analytics_cli::error_report::{ErrorReport, InterruptingError}; + +use crate::unauthorized_error; + +fn error_summary(report: &ErrorReport) -> Vec { + report + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() +} + +fn connection_refused_error() -> anyhow::Error { + anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)) + .context("Failed to create bundle upload") +} + +#[test] +fn prints_a_client_error_explanation_beneath_the_base_message() { + let error = anyhow::anyhow!("HTTP status client error (400 Bad Request)") + .context(ClientErrorExplanation { + code: Some(String::from("SOME_CODE")), + message: Some(String::from("Pass --test-collection-id.")), + }) + .context("Error in create bundle upload endpoint"); + let report = ErrorReport::new( + error, + String::from("fake-org-slug"), + Some(String::from("There was an unexpected error")), + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "There was an unexpected error", + "", + "Pass --test-collection-id.", + "", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + ] + ); +} + +#[test] +fn prints_connection_refused_beneath_the_base_message() { + let report = ErrorReport::new( + connection_refused_error(), + String::from("fake-org-slug"), + Some(String::from("There was an unexpected error")), + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "There was an unexpected error", + "", + "Unable to connect to trunk's server", + ] + ); +} + +#[test] +fn prints_connection_refused_without_a_base_message() { + let report = ErrorReport::new( + connection_refused_error(), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + ["Error Encountered", "Unable to connect to trunk's server"] + ); +} + +#[test] +fn prints_unauthorized_with_a_settings_hint() { + let report = ErrorReport::new(unauthorized_error(), String::from("fake-org-slug"), None); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "", + "", + "Unathorized access, your Trunk organization URL slug or token may be incorrect", + " Hint: You can find it under the settings page at https://app.trunk.io/fake-org-slug/settings", + ] + ); +} + +#[test] +fn prints_a_generic_error_without_a_base_message() { + let report = ErrorReport::new( + anyhow::anyhow!("--allow-forked-pr-uploads requires --test-collection-id"), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "--allow-forked-pr-uploads requires --test-collection-id", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + ] + ); +} + +#[test] +fn prints_an_interrupting_error_with_its_exit_code() { + let report = ErrorReport::new( + anyhow::anyhow!(InterruptingError::new( + "No test output files found to validate" + )), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "No test output files found to validate", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, returning exit code: 70", + ] + ); +} + +#[test] +fn prints_the_default_exit_code() { + let mut report = ErrorReport::new( + anyhow::anyhow!("something went wrong"), + String::from("fake-org-slug"), + None, + ); + report.context.exit_code = Some(exitcode::OK); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "something went wrong", + "For more help, contact us at https://slack.trunk.io/", + "", + "No errors occurred, returning default exit code: 0", + ] + ); +} + +#[test] +fn prints_any_other_exit_code() { + let mut report = ErrorReport::new( + anyhow::anyhow!("something went wrong"), + String::from("fake-org-slug"), + None, + ); + report.context.exit_code = Some(exitcode::USAGE); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "something went wrong", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, returning exit code: 64", + ] + ); +} diff --git a/cli/tests/summary/main.rs b/cli/tests/summary/main.rs new file mode 100644 index 00000000..ad3c87ed --- /dev/null +++ b/cli/tests/summary/main.rs @@ -0,0 +1,12 @@ +mod error_report; +mod upload; +mod validate; + +fn unauthorized_error() -> anyhow::Error { + let response = http::Response::builder().status(401).body("").unwrap(); + anyhow::Error::from( + reqwest::Response::from(response) + .error_for_status() + .unwrap_err(), + ) +} diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs new file mode 100644 index 00000000..8b5c908d --- /dev/null +++ b/cli/tests/summary/upload.rs @@ -0,0 +1,817 @@ +use std::collections::{BTreeMap, HashMap}; + +use api::{client::ClientErrorExplanation, urls::TestCaseGuidScope}; +use bundle::{ + BundleMeta, BundleMetaBaseProps, BundleMetaDebugProps, BundleMetaJunitProps, + QuarantineResolutionMode, Test, TestCollectionProps, +}; +use context::{ + junit::validator::{ + JunitReportValidation, JunitReportValidationIssueSubOptimal, + JunitTestCaseValidationIssueInvalid, JunitTestCaseValidationIssueSubOptimal, + JunitValidationIssue, JunitValidationIssueType, + }, + repo::{BundleRepo, RepoUrlParts}, +}; +use display::end_output::EndOutput; +use pretty_assertions::assert_eq; +use superconsole::Line; +use trunk_analytics_cli::{ + context_quarantine::{QuarantineContext, QuarantineFetchStatus}, + error_report::ErrorReport, + report_limiting::ValidationReport, + upload_command::UploadRunResult, + validate_command::JunitReportValidations, +}; + +use crate::unauthorized_error; + +const BUNDLE_UPLOAD_ID: &str = "4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d"; +const UPLOAD_ERROR_MESSAGE: &str = + "There was an unexpected error that occurred while uploading test results"; + +fn upload_summary(result: &UploadRunResult) -> Vec { + result + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() +} + +fn repo() -> RepoUrlParts { + RepoUrlParts { + host: String::from("github.com"), + owner: String::from("trunk-io"), + name: String::from("analytics-cli"), + } +} + +fn upload_result(num_tests: usize) -> UploadRunResult { + let mut quarantine_context = QuarantineContext::skip_fetch(Vec::new()); + quarantine_context.repo = repo(); + quarantine_context.org_url_slug = String::from("test-org"); + UploadRunResult { + error_report: None, + quarantine_context, + meta: BundleMeta { + base_props: BundleMetaBaseProps { + version: String::new(), + cli_version: String::new(), + org: String::from("test-org"), + test_collection: None, + repo: BundleRepo { + repo: repo(), + ..Default::default() + }, + bundle_upload_id: String::from(BUNDLE_UPLOAD_ID), + tags: Vec::new(), + file_sets: Vec::new(), + envs: HashMap::new(), + upload_time_epoch: 0, + test_command: None, + os_info: None, + quarantined_tests: Vec::new(), + codeowners: None, + use_uncloned_repo: None, + quarantine_resolution_mode: QuarantineResolutionMode::default(), + }, + junit_props: BundleMetaJunitProps { + num_files: 1, + num_tests, + }, + debug_props: BundleMetaDebugProps { + command_line: String::new(), + trunk_envs: HashMap::new(), + }, + bundle_upload_id_v2: String::new(), + variant: None, + internal_bundled_file: None, + failed_tests: Vec::new(), + }, + validations: JunitReportValidations::new(BTreeMap::new()), + validation_report: ValidationReport::Limited, + show_failure_messages: false, + test_collection_short_id: None, + hide_test_collection_links: false, + api_address: String::from("https://api.trunk.io"), + guid_scope: None, + } +} + +fn in_collection(mut result: UploadRunResult) -> UploadRunResult { + result.test_collection_short_id = Some(String::from("abc123")); + result.meta.base_props.test_collection = Some(TestCollectionProps { + short_id: String::from("abc123"), + bundle_meta_id: String::from("0b6f4c1e-8a7d-4e2b-9c3f-5d1a2b3c4d5e"), + bundle_meta_created_at: String::from("2026-10-21T12:00:00Z"), + }); + result +} + +fn failure(name: &str, parent_name: &str, file: Option<&str>) -> Test { + Test { + name: String::from(name), + parent_name: String::from(parent_name), + class_name: None, + file: file.map(String::from), + id: format!("{name}-id"), + timestamp_millis: None, + is_quarantined: false, + failure_message: None, + variant: None, + } +} + +fn warning(issue: JunitTestCaseValidationIssueSubOptimal) -> JunitValidationIssueType { + JunitValidationIssueType::TestCase(JunitValidationIssue::SubOptimal(issue)) +} + +fn report_warning(issue: JunitReportValidationIssueSubOptimal) -> JunitValidationIssueType { + JunitValidationIssueType::Report(JunitValidationIssue::SubOptimal(issue)) +} + +fn name_too_short() -> JunitValidationIssueType { + JunitValidationIssueType::TestCase(JunitValidationIssue::Invalid( + JunitTestCaseValidationIssueInvalid::TestCaseNameTooShort(String::new()), + )) +} + +fn validation(all_issues: Vec) -> anyhow::Result { + Ok(JunitReportValidation { + all_issues, + ..Default::default() + }) +} + +fn validations() -> JunitReportValidations { + JunitReportValidations::new(BTreeMap::from([ + ( + String::from("test-results/a.xml"), + validation(vec![ + name_too_short(), + name_too_short(), + report_warning( + JunitReportValidationIssueSubOptimal::TestCasesFileOrFilepathMissing, + ), + report_warning(JunitReportValidationIssueSubOptimal::MissingTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::FutureTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::OldTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::StaleTimestamps), + warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration), + warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseClassnameTooShort(String::new()), + ), + warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNameTooLong( + String::new(), + )), + warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseFileOrFilepathTooLong( + String::new(), + ), + ), + ]), + ), + ( + String::from("test-results/b.xml"), + Err(anyhow::anyhow!("unexpected end of file")), + ), + ( + String::from("test-results/c.xml"), + validation(vec![name_too_short()]), + ), + ( + String::from("test-results/d.xml"), + validation(vec![warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration, + )]), + ), + (String::from("test-results/e.xml"), validation(Vec::new())), + (String::from("test-results/f.xml"), validation(Vec::new())), + (String::from("test-results/g.xml"), validation(Vec::new())), + (String::from("test-results/h.xml"), validation(Vec::new())), + ( + String::from("test-results/i.xml"), + validation(vec![report_warning( + JunitReportValidationIssueSubOptimal::MissingTimestamps, + )]), + ), + ])) +} + +#[test] +fn prints_a_passing_run() { + let result = upload_result(5); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_that_no_tests_were_found() { + let result = upload_result(0); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 0 Pass: 0 Fail: 0 Quarantined: 0 Pass Ratio: N/A", + "", + "⚠️ No tests were found in the provided test results", + ] + ); +} + +#[test] +fn prints_failures_that_were_all_quarantined() { + let mut result = upload_result(4); + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .group_is_quarantined = true; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![ + failure("adds", "MathTests", Some("src/math.rs")), + failure("subtracts", "MathTests", Some("src/math.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 4 Pass: 2 Fail: 0 Quarantined: 2 Pass Ratio: 50.0%", + "", + "β€οΈβ€πŸ©Ή 2 tests failed and were quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success (0)", + ] + ); +} + +#[test] +fn prints_failures_that_were_not_quarantined() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![failure("adds", "MathTests", Some("src/math.rs"))]; + result.quarantine_context.failures = vec![ + failure("subtracts", "MathTests", Some("src/math.rs")), + failure("divides", "MathTests", Some("src/math.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 2 Fail: 2 Quarantined: 1 Pass Ratio: 40.0%", + "", + "β€οΈβ€πŸ©Ή 1 test failed and was quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "❌ 2 tests failed and were not quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/divides", + " β€· https://app.trunk.io/test-org/flaky-tests/test/divides-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn prints_a_warning_when_the_quarantine_lookup_failed() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.quarantine_context.fetch_status = + QuarantineFetchStatus::FetchFailed(anyhow::anyhow!("request timed out")); + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", + " We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn stops_at_an_error_that_blocks_quarantining() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + unauthorized_error(), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_summary(&result), + [ + "", + "Error Encountered", + "There was an unexpected error that occurred while uploading test results", + "", + "Unathorized access, your Trunk organization URL slug or token may be incorrect", + " Hint: You can find it under the settings page at https://app.trunk.io/test-org/settings", + ] + ); +} + +#[test] +fn proceeds_past_an_upload_error() { + let mut result = upload_result(3); + result.meta.base_props.bundle_upload_id = String::new(); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + anyhow::anyhow!("HTTP status client error (400 Bad Request)") + .context(ClientErrorExplanation { + code: Some(String::from("SOME_CODE")), + message: Some(String::from("Pass --test-collection-id.")), + }) + .context("Error in create bundle upload endpoint"), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_summary(&result), + [ + "", + "Error Encountered", + "There was an unexpected error that occurred while uploading test results", + "", + "Pass --test-collection-id.", + "", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + "Proceeding with test report...", + "", + "πŸ“š Test Report", + " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn prints_a_limited_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "❕ 9 files found, 5 with issues", + "", + " ❌ test-results/a.xml Has Errors And Warnings", + " β†ͺ ❌ Errors", + " β†ͺ test case name too short", + " β†ͺ test case name too short", + " β†ͺ ⚠️ Warnings", + " β†ͺ report has test cases with missing file or filepath", + " β†ͺ report has test cases with missing timestamp", + " β†ͺ report has test cases with future timestamp", + " β†ͺ report has old (> 24 hour(s)) timestamps", + " β†ͺ report has stale (> 1 hour(s)) timestamps", + " β†ͺ test case or parent has no time duration", + " β†ͺ test case classname too short", + " β†ͺ test case name too long, truncated to 1000", + " ❌ test-results/c.xml Has Errors", + " β†ͺ test case name too short", + " ⚠️ test-results/d.xml Has Warnings", + " β†ͺ test case or parent has no time duration", + "…and 1 more", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_a_full_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + result.validation_report = ValidationReport::Full; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "❕ 9 files found, 5 with issues", + "", + " ❌ test-results/a.xml Has Errors And Warnings", + " β†ͺ ❌ Errors", + " β†ͺ test case name too short", + " β†ͺ test case name too short", + " β†ͺ ⚠️ Warnings", + " β†ͺ report has test cases with missing file or filepath", + " β†ͺ report has test cases with missing timestamp", + " β†ͺ report has test cases with future timestamp", + " β†ͺ report has old (> 24 hour(s)) timestamps", + " β†ͺ report has stale (> 1 hour(s)) timestamps", + " β†ͺ test case or parent has no time duration", + " β†ͺ test case classname too short", + " β†ͺ test case name too long, truncated to 1000", + " β†ͺ test case file or filepath too long", + " ❌ test-results/c.xml Has Errors", + " β†ͺ test case name too short", + " ⚠️ test-results/d.xml Has Warnings", + " β†ͺ test case or parent has no time duration", + " ⚠️ test-results/i.xml Has Warnings", + " β†ͺ report has test cases with missing timestamp", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_a_blank_line_for_no_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + result.validation_report = ValidationReport::None; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_validation_when_every_file_is_correct() { + let mut result = upload_result(5); + result.validations = JunitReportValidations::new(BTreeMap::from([( + String::from("test-results/a.xml"), + validation(Vec::new()), + )])); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "βœ… 1 file found, all fully correct", + "", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_long_failure_messages_when_asked() { + let mut result = upload_result(1); + result.show_failure_messages = true; + result.quarantine_context.exit_code = 1; + let mut message_lines = vec![ + String::from("assertion `left == right` failed\r"), + String::from("\tleft: 1"), + String::new(), + String::from(" "), + String::from("\tright: 2"), + ]; + message_lines.extend((6..=23).map(|n| format!("frame {n}"))); + result.quarantine_context.failures = vec![Test { + failure_message: Some(message_lines.join("\n")), + ..failure("adds", "MathTests", None) + }]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 1 Pass: 0 Fail: 1 Quarantined: 0 Pass Ratio: 0.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " Failure: ", + " assertion `left == right` failed", + " left: 1", + " right: 2", + " frame 6", + " frame 7", + " frame 8", + " frame 9", + " frame 10", + " frame 11", + " frame 12", + " frame 13", + " frame 14", + " frame 15", + " frame 16", + " frame 17", + " frame 18", + " frame 19", + " frame 20", + " …and 3 more lines not shown", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn prints_the_upload_as_a_collection_link() { + let mut result = in_collection(upload_result(2)); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![Test { + id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), + ..failure("adds", "MathTests", None) + }]; + result.guid_scope = Some(TestCaseGuidScope { + test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), + repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), + }); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/tests/81d0929e-fc06-8b61-8506-991d8ebc51ea", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn links_tests_by_short_link_without_collection_ids() { + let mut result = in_collection(upload_result(2)); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn prints_the_bundle_upload_id_when_collection_links_are_hidden() { + let mut result = in_collection(upload_result(2)); + result.hide_test_collection_links = true; + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![Test { + id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), + ..failure("adds", "MathTests", None) + }]; + result.guid_scope = Some(TestCaseGuidScope { + test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), + repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), + }); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/c33a7f64-8f3e-5db9-b37b-2ea870d2441b?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn omits_the_upload_line_without_a_bundle_upload_id() { + let mut result = upload_result(5); + result.meta.base_props.bundle_upload_id = String::new(); + + assert_eq!( + upload_summary(&result), + [ + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn groups_failures_by_file_then_suite_then_neither() { + let mut result = upload_result(10); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![ + failure("parses_dates", "DateTests", Some("src/date.rs")), + failure("adds", "MathTests", None), + failure("loads", "", None), + failure("parses_times", "TimeTests", Some("src/date.rs")), + failure("subtracts", "MathTests", None), + failure("boots", "", None), + failure("formats", "", Some("src/a.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 10 Pass: 3 Fail: 7 Quarantined: 0 Pass Ratio: 30.0%", + "", + "❌ 7 tests failed and were not quarantined", + "", + " πŸ“ src/a.rs", + " formats", + " β€· https://app.trunk.io/test-org/flaky-tests/test/formats-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ“ src/date.rs", + " DateTests/parses_dates", + " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_dates-id?repo=trunk-io%2Fanalytics-cli", + " TimeTests/parses_times", + " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_times-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ” Other", + " loads", + " β€· https://app.trunk.io/test-org/flaky-tests/test/loads-id?repo=trunk-io%2Fanalytics-cli", + " boots", + " β€· https://app.trunk.io/test-org/flaky-tests/test/boots-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn shows_three_failures_per_group() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = ["adds", "subtracts", "multiplies", "divides", "rounds"] + .into_iter() + .map(|name| failure(name, "MathTests", Some("src/math.rs"))) + .collect(); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 0 Fail: 5 Quarantined: 0 Pass Ratio: 0.0%", + "", + "❌ 5 tests failed and were not quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/multiplies", + " β€· https://app.trunk.io/test-org/flaky-tests/test/multiplies-id?repo=trunk-io%2Fanalytics-cli", + " …and 2 more failures in this group", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn warns_about_a_non_zero_exit_code_without_failures() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "⚠️ No test failures found, but non zero exit code provided: 1", + "This may indicate that some tests were not run or that there were other issues during the test run.", + ] + ); +} diff --git a/cli/tests/summary/validate.rs b/cli/tests/summary/validate.rs new file mode 100644 index 00000000..f5ff3cbb --- /dev/null +++ b/cli/tests/summary/validate.rs @@ -0,0 +1,110 @@ +use clap::Parser; +use display::end_output::EndOutput; +use pretty_assertions::assert_eq; +use superconsole::Line; +use trunk_analytics_cli::validate_command::{ValidateArgs, run_validate}; + +#[derive(Parser)] +struct Validate { + #[command(flatten)] + args: ValidateArgs, +} + +async fn validate_summary(args: &[&str]) -> Vec { + let Validate { args } = Validate::parse_from(["validate"].iter().chain(args)); + run_validate(args) + .await + .unwrap() + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + // CODEOWNERS is printed as an absolute path. + .map(|line| line.replace(env!("CARGO_MANIFEST_DIR"), "")) + .collect() +} + +#[tokio::test] +async fn prints_validation_errors_warnings_and_codeowners() { + assert_eq!( + validate_summary(&[ + "--junit-paths", + "test_fixtures/summary/validate/*.xml", + "--codeowners-path", + "test_fixtures/summary/validate", + ]) + .await, + [ + "πŸ“š Validating 4 files:", + " Files matching: test_fixtures/summary/validate/*.xml:", + " test_fixtures/summary/validate/broken.xml", + " test_fixtures/summary/validate/empty.xml", + " test_fixtures/summary/validate/multiple.xml", + " test_fixtures/summary/validate/unit.xml", + "", + "File parse issues", + "test_fixtures/summary/validate/broken.xml", + "1 errors, 0 warnings", + " ❌ Errors:", + " ill-formed document: expected ``, but `` was foundCaused by: expected ``, but `` was found", + "", + "test_fixtures/summary/validate/empty.xml", + "0 errors, 1 warnings", + " ⚠️ Warnings:", + " no reports found", + "", + "test_fixtures/summary/validate/multiple.xml", + "1 errors, 0 warnings", + " ❌ Errors:", + " multiple reports found", + "", + "Test Validation Issues", + "test_fixtures/summary/validate/unit.xml", + "1 test suites, 3 test cases, 1 errors, 3 warnings", + " ❌ Errors:", + " test case name too short", + " ⚠️ Warnings:", + " report has test cases with missing file or filepath", + " report has test cases with missing timestamp", + " test case or parent has no time duration", + "", + "Checking for codeowners file...", + " Found codeowners path: \"/test_fixtures/summary/validate/CODEOWNERS\"", + " CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", + "", + "0 valid files, 2 files with warnings, and 3 files with errors, with 4 files total (a file is double counted if it has both errors and warnings)", + "4 warnings, and 3 errors", + ] + ); +} + +#[tokio::test] +async fn prints_bep_parse_errors_without_a_codeowners_file() { + assert_eq!( + validate_summary(&[ + "--bazel-bep-path", + "test_fixtures/summary/bep/bep_with_parse_error" + ]) + .await, + [ + "⚠️ BEP file had parse errors: [\"Error parsing build event: EOF while parsing a value at line 3 column 0\"]", + "", + "πŸ“š Validating 1 files:", + " Files matching: test_fixtures/summary/bep/test.xml:", + " test_fixtures/summary/bep/test.xml", + "", + "File parse issues", + "Test Validation Issues", + "test_fixtures/summary/bep/test.xml", + "1 test suites, 1 test cases, 0 errors, 1 warnings", + " ⚠️ Warnings:", + " report has test cases with missing timestamp", + "", + "Checking for codeowners file...", + " No codeowners file found", + "", + "0 valid files, 1 file with warnings, and 0 files with errors, with 1 file total (a file is double counted if it has both errors and warnings)", + "1 warning, and 0 errors", + ] + ); +} From b4f7209d5542ba216d99b219267a329d72313556 Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 21 Oct 2026 23:51:08 +0000 Subject: [PATCH 03/19] refactor(cli): build the upload summary as data, separate from rendering UploadReport, in the new report module, works out everything the upload summary says once: counts, failure groups, links, validation results, the closing message (Outcome) and the exit code. The terminal output is now a text renderer over that report plus TextOptions (--validation-report and --show-failure-messages), and main exits with report.exit_code. The report derives Serialize, for the JSON summary to come. ErrorReport now classifies its error once as an ErrorKind; ErrorSummary holds what gets printed. run_upload collects deprecation warnings and BEP parse errors as notices, still only logged. The lock-down tests pass with only their helpers changed. `test` keeps exiting with the test run's exit code when the upload fails, so UploadReport::new takes the command. Refs TRUNK-22401 Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 1 + cli/Cargo.toml | 1 + cli/src/error_report.rs | 197 ++++-------- cli/src/lib.rs | 1 + cli/src/main.rs | 42 ++- cli/src/report/error.rs | 36 +++ cli/src/report/mod.rs | 11 + cli/src/report/notice.rs | 31 ++ cli/src/report/text.rs | 506 ++++++++++++++++++++++++++++++ cli/src/report/upload.rs | 271 ++++++++++++++++ cli/src/upload_command.rs | 420 ++----------------------- cli/src/validate_command.rs | 175 +---------- cli/tests/summary/error_report.rs | 9 +- cli/tests/summary/upload.rs | 44 ++- 14 files changed, 1014 insertions(+), 731 deletions(-) create mode 100644 cli/src/report/error.rs create mode 100644 cli/src/report/mod.rs create mode 100644 cli/src/report/notice.rs create mode 100644 cli/src/report/text.rs create mode 100644 cli/src/report/upload.rs diff --git a/Cargo.lock b/Cargo.lock index d7d3a3cf..b391e8a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5921,6 +5921,7 @@ dependencies = [ "rstest", "sentry", "sentry-tracing", + "serde", "serde_json", "superconsole", "tempfile", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b6cf076b..5455215c 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -50,6 +50,7 @@ sentry-tracing = "0.36.0" openssl = { version = "0.10.78", features = ["vendored"] } openssl-src = { version = "=300.3.1", optional = true } quick-junit = "0.5.0" +serde = { version = "1.0.215", features = ["derive"] } serde_json = "1.0" clap-verbosity-flag = "3.0.2" proto = { path = "../proto" } diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index c13284f2..405b6702 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -1,25 +1,17 @@ -use api::client::{ClientErrorExplanation, get_api_host}; -use display::end_output::EndOutput; +use api::client::ClientErrorExplanation; use http::StatusCode; -use superconsole::{ - Line, Span, - style::{Attribute, Stylize, style}, -}; +use serde::Serialize; -const HELP_TEXT: &str = "For more help, contact us at https://slack.trunk.io/"; -const CONNECTION_REFUSED_CONTEXT: &str = concat!("Unable to connect to trunk's server",); +pub(crate) const HELP_TEXT: &str = "For more help, contact us at https://slack.trunk.io/"; +pub(crate) const CONNECTION_REFUSED_CONTEXT: &str = concat!("Unable to connect to trunk's server",); pub(crate) const UNAUTHORIZED_CONTEXT: &str = concat!("Unathorized access, your Trunk organization URL slug or token may be incorrect",); const GIX_ERROR_CONTEXT: &str = "Unable to open git repository"; -fn add_settings_url_to_context(domain: String, org_url_slug: &String) -> String { - let settings_url = format!("{}/{}/settings", domain.replace("api", "app"), org_url_slug); - format!( - "Hint: You can find it under the settings page at {}", - settings_url - ) +pub(crate) fn settings_url(domain: &str, org_url_slug: &str) -> String { + format!("{}/{}/settings", domain.replace("api", "app"), org_url_slug) } pub struct InterruptingError { @@ -53,50 +45,77 @@ pub struct Context { pub struct ErrorReport { pub error: anyhow::Error, pub context: Context, + pub kind: ErrorKind, } impl ErrorReport { pub fn new(error: anyhow::Error, org_url_slug: String, base_message: Option) -> Self { + let kind = ErrorKind::of(&error); + match kind { + ErrorKind::ConnectionRefused => tracing::warn!(CONNECTION_REFUSED_CONTEXT), + ErrorKind::Unauthorized => tracing::warn!(UNAUTHORIZED_CONTEXT), + ErrorKind::GitRepository => tracing::warn!(GIX_ERROR_CONTEXT), + ErrorKind::Interrupted => { + if let Some(message) = get_interrupting_message(&error) { + tracing::warn!("{}", message); + } + } + ErrorKind::Other => { + tracing::error!("{}", error); + tracing::error!(hidden_in_console = true, "Caused by error: {:#?}", error); + } + } Self { context: Context { base_message, org_url_slug, - exit_code: ErrorReport::find_exit_code(&error), + exit_code: kind.exit_code(), }, error, + kind, } } +} - fn find_exit_code(error: &anyhow::Error) -> Option { - if is_connection_refused(error) { - tracing::warn!(CONNECTION_REFUSED_CONTEXT); - return None; - } - - if is_unauthorized(error) { - tracing::warn!(UNAUTHORIZED_CONTEXT); - return Some(exitcode::SOFTWARE); - } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum ErrorKind { + ConnectionRefused, + Unauthorized, + GitRepository, + Interrupted, + Other, +} - if is_gix_error(error) { - tracing::warn!(GIX_ERROR_CONTEXT); - return Some(exitcode::SOFTWARE); +impl ErrorKind { + pub fn of(error: &anyhow::Error) -> Self { + if is_connection_refused(error) { + Self::ConnectionRefused + } else if is_unauthorized(error) { + Self::Unauthorized + } else if is_gix_error(error) { + Self::GitRepository + } else if get_interrupting_message(error).is_some() { + Self::Interrupted + } else { + Self::Other } + } - if let Some(message) = get_interrupting_message(error) { - tracing::warn!("{}", message); - return Some(exitcode::SOFTWARE); + pub fn exit_code(self) -> Option { + match self { + Self::ConnectionRefused | Self::Other => None, + Self::Unauthorized | Self::GitRepository | Self::Interrupted => { + Some(exitcode::SOFTWARE) + } } - - tracing::error!("{}", error); - tracing::error!(hidden_in_console = true, "Caused by error: {:#?}", error); - None } - pub fn should_block_quarantining(&self) -> bool { - is_unauthorized(&self.error) - || is_gix_error(&self.error) - || get_interrupting_message(&self.error).is_some() + pub fn blocks_quarantining(self) -> bool { + matches!( + self, + Self::Unauthorized | Self::GitRepository | Self::Interrupted + ) } } @@ -137,7 +156,7 @@ fn is_gix_error(error: &anyhow::Error) -> bool { } } -fn get_client_error_explanation(error: &anyhow::Error) -> Option { +pub(crate) fn get_client_error_explanation(error: &anyhow::Error) -> Option { error .downcast_ref::() .map(ToString::to_string) @@ -150,102 +169,10 @@ fn get_interrupting_message(error: &anyhow::Error) -> Option { .map(|e| e.message.clone()) } -impl EndOutput for ErrorReport { - fn output(&self) -> anyhow::Result> { - let Context { - base_message, - org_url_slug, - exit_code, - } = &self.context; - let mut lines = Vec::new(); - lines.push(Line::from_iter([Span::new_styled( - String::from("Error Encountered").attribute(Attribute::Bold), - )?])); - if is_connection_refused(&self.error) { - if let Some(base_message) = base_message { - lines.push(Line::from_iter([Span::new_unstyled_lossy(base_message)])); - lines.push(Line::default()); - } - lines.push(Line::from_iter([Span::new_unstyled( - CONNECTION_REFUSED_CONTEXT, - )?])); - return Ok(lines); - } - let api_host = get_api_host(); - if is_unauthorized(&self.error) { - { - lines.extend(vec![ - Line::from_iter([Span::new_unstyled_lossy( - base_message.as_deref().unwrap_or_default(), - )]), - Line::default(), - Line::from_iter([Span::new_unstyled(UNAUTHORIZED_CONTEXT)?]), - Line::from_iter([Span::new_unstyled_lossy(add_settings_url_to_context( - api_host, - org_url_slug, - ))]), - ]); - if let Some(line) = lines.last_mut() { - line.pad_left(2); - } - return Ok(lines); - } - } - if base_message.is_some() { - lines.push(Line::from_iter([Span::new_unstyled_lossy( - base_message.as_deref().unwrap_or("An error occurred"), - )])); - lines.push(Line::default()); - if let Some(explanation) = get_client_error_explanation(&self.error) { - lines.push(Line::from_iter([Span::new_unstyled_lossy(explanation)])); - lines.push(Line::default()); - } - } else { - lines.push(Line::from_iter([Span::new_unstyled_lossy( - self.error.to_string(), - )])); - } - lines.push(Line::from_iter([Span::new_unstyled(HELP_TEXT)?])); - lines.push(Line::default()); - match exit_code { - Some(exitcode::OK) => lines.push(Line::from_iter([ - Span::new_unstyled("No errors occurred, returning default exit code: ")?, - Span::new_styled(style(exitcode::OK.to_string()).attribute(Attribute::Bold))?, - ])), - Some(exitcode::SOFTWARE) => { - // SOFTWARE is used to indicate that the upload command failed due to user error - lines.push(Line::from_iter([ - Span::new_unstyled("Errors occurred during execution, returning exit code: ")?, - Span::new_styled( - style(exitcode::SOFTWARE.to_string()).attribute(Attribute::Bold), - )?, - ])); - } - Some(other_code) => { - // Should be an unused codepath, but we log it for completeness - lines.push(Line::from_iter([ - Span::new_unstyled("Errors occurred during execution, returning exit code: ")?, - Span::new_styled(style(other_code.to_string()).attribute(Attribute::Bold))?, - ])); - } - None => { - // If uploads fail because trunk is down, we fall back to whatever came out of quarantining - // to minimize customer impact - lines.push(Line::from_iter([Span::new_unstyled( - "Errors occurred during execution, using quarantining exit code", - )?])); - } - } - Ok(lines) - } -} - #[test] -fn adds_settings_if_domain_present() { - let host = "https://api.fake-trunk.io"; - let final_context = add_settings_url_to_context(host.into(), &String::from("fake-org-slug")); +fn builds_the_settings_url_from_the_api_host() { assert_eq!( - final_context, - "Hint: You can find it under the settings page at https://app.fake-trunk.io/fake-org-slug/settings" + settings_url("https://api.fake-trunk.io", "fake-org-slug"), + "https://app.fake-trunk.io/fake-org-slug/settings" ) } diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 92ce0bd4..e7e10ab9 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -2,6 +2,7 @@ pub mod context; pub mod context_quarantine; pub mod error_report; pub mod print; +pub mod report; pub mod report_limiting; pub mod test_command; pub mod upload_command; diff --git a/cli/src/main.rs b/cli/src/main.rs index 1aaa755c..e70e2aec 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -17,6 +17,10 @@ use tracing_subscriber::{filter::FilterFn, prelude::*}; use trunk_analytics_cli::{ context::gather_debug_props, error_report::ErrorReport, + report::{ + ErrorSummary, UploadCommand, UploadReport, + text::{TextOptions, UploadText}, + }, test_command::{TestArgs, run_test}, upload_command::{RunUploadOptions, UploadArgs, UploadRunResult, run_upload}, validate_command::{ValidateArgs, ValidateRunResult, run_validate}, @@ -148,31 +152,28 @@ fn main() -> anyhow::Result<()> { close_out_and_exit(exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(RunResult::Upload(run_result))) => { - let result_ptr = Arc::new(run_result); + let text = UploadText { + report: UploadReport::new(&run_result, UploadCommand::Upload), + options: TextOptions::from(&run_result), + }; + let exit_code = text.report.exit_code; send_message( - DisplayMessage::Final(result_ptr.clone(), String::from("upload display")), + DisplayMessage::Final(Arc::new(text), String::from("upload display")), &render_sender, ); - let exit_code = result_ptr - .error_report - .as_ref() - .map(|e| e.context.exit_code) - .flatten() - .unwrap_or(result_ptr.quarantine_context.exit_code); close_out_and_exit(exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(RunResult::Test(run_result))) => { - let result_ptr = Arc::new(run_result); + let text = UploadText { + report: UploadReport::new(&run_result, UploadCommand::Test), + options: TextOptions::from(&run_result), + }; + let exit_code = text.report.exit_code; send_message( - DisplayMessage::Final(result_ptr.clone(), String::from("test display")), + DisplayMessage::Final(Arc::new(text), String::from("test display")), &render_sender, ); - close_out_and_exit( - result_ptr.quarantine_context.exit_code, - guard, - render_sender, - render_handle, - ) + close_out_and_exit(exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(RunResult::Validate(run_result))) => { let result_ptr = Arc::new(run_result); @@ -183,13 +184,10 @@ fn main() -> anyhow::Result<()> { close_out_and_exit(result_ptr.exit_code(), guard, render_sender, render_handle) } MainOutcome::Finished(Err(error)) => { - let error_report = ErrorReport::new(error, org_url_slug, None); - let exit_code = error_report.context.exit_code.unwrap_or(exitcode::OK); + let error = ErrorSummary::new(&ErrorReport::new(error, org_url_slug, None)); + let exit_code = error.exit_code.unwrap_or(exitcode::OK); send_message( - DisplayMessage::Final( - Arc::new(error_report), - String::from("error display"), - ), + DisplayMessage::Final(Arc::new(error), String::from("error display")), &render_sender, ); close_out_and_exit(exit_code, guard, render_sender, render_handle) diff --git a/cli/src/report/error.rs b/cli/src/report/error.rs new file mode 100644 index 00000000..316d2b16 --- /dev/null +++ b/cli/src/report/error.rs @@ -0,0 +1,36 @@ +use api::client::get_api_host; +use serde::Serialize; + +use crate::error_report::{ErrorKind, ErrorReport, get_client_error_explanation, settings_url}; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ErrorSummary { + pub kind: ErrorKind, + pub message: Option, + pub error: String, + pub causes: Vec, + pub explanation: Option, + pub settings_url: Option, + pub exit_code: Option, +} + +impl ErrorSummary { + pub fn new(report: &ErrorReport) -> Self { + let ErrorReport { + error, + context, + kind, + } = report; + Self { + kind: *kind, + message: context.base_message.clone(), + error: error.to_string(), + causes: error.chain().skip(1).map(ToString::to_string).collect(), + explanation: get_client_error_explanation(error), + settings_url: (*kind == ErrorKind::Unauthorized) + .then(|| settings_url(&get_api_host(), &context.org_url_slug)), + exit_code: context.exit_code, + } + } +} diff --git a/cli/src/report/mod.rs b/cli/src/report/mod.rs new file mode 100644 index 00000000..3958ec5b --- /dev/null +++ b/cli/src/report/mod.rs @@ -0,0 +1,11 @@ +mod error; +mod notice; +pub mod text; +mod upload; + +pub use error::ErrorSummary; +pub use notice::{Notice, NoticeKind}; +pub use upload::{ + FailedTest, FailureGroup, FileValidation, GroupKey, Outcome, TestCounts, UploadCommand, + UploadReport, +}; diff --git a/cli/src/report/notice.rs b/cli/src/report/notice.rs new file mode 100644 index 00000000..763c6bec --- /dev/null +++ b/cli/src/report/notice.rs @@ -0,0 +1,31 @@ +use serde::Serialize; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Notice { + pub kind: NoticeKind, + pub message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum NoticeKind { + Deprecation, + BepParseError, +} + +impl Notice { + pub fn deprecation>(message: T) -> Self { + Self { + kind: NoticeKind::Deprecation, + message: message.into(), + } + } + + pub fn bep_parse_error>(message: T) -> Self { + Self { + kind: NoticeKind::BepParseError, + message: message.into(), + } + } +} diff --git a/cli/src/report/text.rs b/cli/src/report/text.rs new file mode 100644 index 00000000..19f867b9 --- /dev/null +++ b/cli/src/report/text.rs @@ -0,0 +1,506 @@ +use constants::EXIT_SUCCESS; +use display::end_output::EndOutput; +use pluralizer::pluralize; +use superconsole::{ + Hyperlink, Line, Lines, Span, SuperConsole, + style::{Attribute, Color, Stylize, style}, +}; + +use super::{ + ErrorSummary, FailureGroup, FileValidation, GroupKey, Outcome, TestCounts, UploadReport, +}; +use crate::{ + error_report::{CONNECTION_REFUSED_CONTEXT, ErrorKind, HELP_TEXT, UNAUTHORIZED_CONTEXT}, + report_limiting::ValidationReport, + upload_command::{UploadRunResult, get_bundle_upload_id_message}, +}; + +/// Flags that change how much of the summary to show, not what it says. +#[derive(Debug, Clone)] +pub struct TextOptions { + pub validation_report: ValidationReport, + // TODO: show_failure_messages is a temporary flag to show failure messages + // in the output. It should be removed once we are confident in this flow + // and we should use the validation report flag instead. + pub show_failure_messages: bool, +} + +impl From<&UploadRunResult> for TextOptions { + fn from(result: &UploadRunResult) -> Self { + Self { + validation_report: result.validation_report.clone(), + show_failure_messages: result.show_failure_messages, + } + } +} + +pub struct UploadText { + pub report: UploadReport, + pub options: TextOptions, +} + +impl EndOutput for UploadText { + fn output(&self) -> anyhow::Result> { + render_upload(&self.report, &self.options) + } +} + +impl EndOutput for ErrorSummary { + fn output(&self) -> anyhow::Result> { + render_error(self) + } +} + +pub fn render_error(error: &ErrorSummary) -> anyhow::Result> { + let mut lines = Vec::new(); + lines.push(Line::from_iter([Span::new_styled( + String::from("Error Encountered").attribute(Attribute::Bold), + )?])); + match error.kind { + ErrorKind::ConnectionRefused => { + if let Some(message) = &error.message { + lines.push(Line::from_iter([Span::new_unstyled_lossy(message)])); + lines.push(Line::default()); + } + lines.push(Line::from_iter([Span::new_unstyled( + CONNECTION_REFUSED_CONTEXT, + )?])); + return Ok(lines); + } + ErrorKind::Unauthorized => { + let mut hint = Line::from_iter([Span::new_unstyled_lossy(format!( + "Hint: You can find it under the settings page at {}", + error.settings_url.as_deref().unwrap_or_default() + ))]); + hint.pad_left(2); + lines.extend([ + Line::from_iter([Span::new_unstyled_lossy( + error.message.as_deref().unwrap_or_default(), + )]), + Line::default(), + Line::from_iter([Span::new_unstyled(UNAUTHORIZED_CONTEXT)?]), + hint, + ]); + return Ok(lines); + } + ErrorKind::GitRepository | ErrorKind::Interrupted | ErrorKind::Other => {} + } + if let Some(message) = &error.message { + lines.push(Line::from_iter([Span::new_unstyled_lossy(message)])); + lines.push(Line::default()); + if let Some(explanation) = &error.explanation { + lines.push(Line::from_iter([Span::new_unstyled_lossy(explanation)])); + lines.push(Line::default()); + } + } else { + lines.push(Line::from_iter([Span::new_unstyled_lossy(&error.error)])); + } + lines.push(Line::from_iter([Span::new_unstyled(HELP_TEXT)?])); + lines.push(Line::default()); + match error.exit_code { + Some(exitcode::OK) => lines.push(Line::from_iter([ + Span::new_unstyled("No errors occurred, returning default exit code: ")?, + Span::new_styled(style(exitcode::OK.to_string()).attribute(Attribute::Bold))?, + ])), + Some(exit_code) => lines.push(Line::from_iter([ + Span::new_unstyled("Errors occurred during execution, returning exit code: ")?, + Span::new_styled(style(exit_code.to_string()).attribute(Attribute::Bold))?, + ])), + // If uploads fail because trunk is down, we fall back to whatever came out of quarantining + // to minimize customer impact + None => lines.push(Line::from_iter([Span::new_unstyled( + "Errors occurred during execution, using quarantining exit code", + )?])), + } + Ok(lines) +} + +pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Result> { + let mut output: Vec = Vec::new(); + if let Some(error) = &report.error { + output.push(Line::default()); + output.extend(render_error(error)?); + if error.kind.blocks_quarantining() { + return Ok(output); + } + output.push(Line::from_iter([Span::new_unstyled( + "Proceeding with test report...", + )?])); + output.push(Line::default()); + } + + let bold = |text: String| Span::new_styled(style(text).attribute(Attribute::Bold)); + let upload_line = match (&report.upload_url, &report.bundle_upload_id) { + (Some(url), _) => Some(Line::from_iter([ + bold(String::from("🏷️ Upload: "))?, + hyperlink_span(url.clone(), Attribute::Bold)?, + ])), + (None, Some(bundle_upload_id)) => Some(Line::from_iter([bold( + get_bundle_upload_id_message(bundle_upload_id), + )?])), + (None, None) => None, + }; + if let Some(line) = upload_line { + output.push(line); + output.push(Line::default()); + } + + if !report.validations.is_empty() { + output.extend(render_validations( + &report.validations, + &options.validation_report, + )?); + output.push(Line::default()); + } + + let TestCounts { + total, + passed, + failed, + quarantined, + pass_ratio, + } = report.counts; + let pass_ratio = pass_ratio.map_or_else( + || String::from("N/A"), + |pass_ratio| format!("{:.1}%", pass_ratio * 100.0), + ); + output.push(Line::from_iter([Span::new_styled( + style("πŸ“š Test Report".to_string()).attribute(Attribute::Bold), + )?])); + output.push(Line::from_iter([Span::new_unstyled(format!( + " Total: {} Pass: {} Fail: {} Quarantined: {} Pass Ratio: {}", + total, passed, failed, quarantined, pass_ratio + ))?])); + if report.quarantine_lookup_failed { + output.push(Line::from_iter([Span::new_styled( + style(" We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures".to_string()).with(Color::Yellow).attribute(Attribute::Bold), + )?])); + } + output.push(Line::default()); + + if quarantined > 0 { + output.extend(vec![ + Line::from_iter([ + Span::new_unstyled("β€οΈβ€πŸ©Ή ")?, + Span::new_styled( + style(format!( + "{} ", + pluralize("test", quarantined as isize, true) + )) + .attribute(Attribute::Bold), + )?, + Span::new_styled(style(format!( + "failed and {} ", + pluralize("was", quarantined as isize, false) + )))?, + Span::new_styled( + style(String::from("quarantined")) + .with(Color::Yellow) + .attribute(Attribute::Bold), + )?, + ]), + Line::default(), + ]); + output.extend(render_groups(&report.quarantined, options)?); + } + + if failed > 0 { + output.extend(vec![ + Line::from_iter([ + Span::new_unstyled("❌ ")?, + Span::new_unstyled(format!("{} ", pluralize("test", failed as isize, true)))?, + Span::new_styled( + style(String::from("failed ")) + .with(Color::Red) + .attribute(Attribute::Bold), + )?, + Span::new_styled(style(format!( + "and {} ", + pluralize("was", failed as isize, false) + )))?, + Span::new_styled(style(String::from("not ")).attribute(Attribute::Bold))?, + Span::new_unstyled("quarantined")?, + ]), + Line::default(), + ]); + output.extend(render_groups(&report.failures, options)?); + } + + match report.outcome { + Outcome::NoTestsFound => output.push(Line::from_iter([Span::new_unstyled( + "⚠️ No tests were found in the provided test results", + )?])), + Outcome::AllFailuresQuarantined => output.push(Line::from_iter([ + Span::new_unstyled( + "πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success ", + )?, + Span::new_styled(style(format!("({})", EXIT_SUCCESS)).attribute(Attribute::Bold))?, + ])), + Outcome::NoFailures => output.push(Line::from_iter([Span::new_unstyled( + "πŸŽ‰ No test failures found!", + )?])), + Outcome::NoFailuresButNonZeroExitCode => { + output.push(Line::from_iter([ + Span::new_unstyled( + "⚠️ No test failures found, but non zero exit code provided: ", + )?, + Span::new_styled( + style(format!("{}", report.exit_code)).attribute(Attribute::Bold), + )?, + ])); + output.push(Line::from_iter([Span::new_unstyled( + "This may indicate that some tests were not run or that there were other issues during the test run.", + )?])); + } + Outcome::FailuresNotQuarantined => output.push(Line::from_iter([ + Span::new_unstyled( + "⚠️ Some test failures were not quarantined, using exit code: ".to_string(), + )?, + Span::new_styled(style(format!("{}", report.exit_code)).attribute(Attribute::Bold))?, + ])), + } + Ok(output) +} + +fn render_groups(groups: &[FailureGroup], options: &TextOptions) -> anyhow::Result { + let mut output = Vec::new(); + for group in groups { + let (header, color) = match &group.key { + GroupKey::File(file) => (format!("πŸ“ {}", file), Color::Cyan), + GroupKey::Suite(suite) => (format!("πŸ“¦ {}", suite), Color::Magenta), + GroupKey::Other => (String::from("πŸ” Other"), Color::Yellow), + }; + output.push(Line::from_iter([Span::new_styled( + style(header).with(color).attribute(Attribute::Bold), + )?])); + for test in group.tests.iter().take(3) { + let output_name = format!( + "{}{}{}", + test.suite, + if test.suite.is_empty() { "" } else { "/" }, + test.name + ); + let mut test_line = Line::from_iter([Span::new_styled_lossy( + style(output_name).attribute(Attribute::Bold), + )]); + test_line.pad_left(2); + output.push(test_line); + // A test without a link has always failed the whole summary. + let link = test + .link + .clone() + .ok_or_else(|| anyhow::anyhow!("Unable to link to test {}", test.id))?; + let mut link_output = Line::from_iter([ + Span::new_unstyled("β€· ")?, + hyperlink_span(link, Attribute::Underlined)?, + ]); + link_output.pad_left(4); + output.push(link_output); + if let Some(failure_message) = test + .failure_message + .as_ref() + .filter(|_| options.show_failure_messages) + { + output.extend(render_failure_message(failure_message)?); + } + } + if group.tests.len() > 3 { + let mut more_failures = Line::from_iter([Span::new_unstyled(format!( + "…and {} more failures in this group", + group.tests.len() - 3 + ))?]); + more_failures.pad_left(2); + output.push(more_failures); + } + output.push(Line::default()); + } + let mut output = Lines(output); + output.pad_lines_left(2); + output.pad_lines_bottom(1); + Ok(output) +} + +fn render_failure_message(failure_message: &str) -> anyhow::Result> { + let mut output = Vec::new(); + let lines: Vec<&str> = failure_message.split('\n').collect(); + let max_lines = 20; + let mut failure_header = Line::from_iter([Span::new_styled( + style("Failure: ".to_string()).with(Color::DarkGrey), + )?]); + failure_header.pad_left(4); + output.push(failure_header); + for (j, line) in lines.iter().take(max_lines).enumerate() { + let sanitized_line = line.replace('\t', " ").replace('\r', ""); + if !sanitized_line.trim().is_empty() || j == 0 { + let mut failure_output = Line::from_iter([ + Span::new_unstyled(" ")?, + Span::new_styled_lossy( + style(sanitized_line) + .with(Color::Grey) + .attribute(Attribute::Italic), + ), + ]); + failure_output.pad_left(4); + output.push(failure_output); + } + } + if lines.len() > max_lines { + let omitted = lines.len() - max_lines; + let mut more_output = Line::from_iter([ + Span::new_unstyled(" ")?, + Span::new_styled( + style(format!("…and {omitted} more lines not shown")) + .with(Color::Grey) + .attribute(Attribute::Italic), + )?, + ]); + more_output.pad_left(4); + output.push(more_output); + } + Ok(output) +} + +fn render_validations( + validations: &[FileValidation], + limits: &ValidationReport, +) -> anyhow::Result> { + let mut output: Vec = Vec::new(); + if limits == &ValidationReport::None { + return Ok(output); + } + + output.push(Line::from_iter([Span::new_styled( + String::from("πŸ“‚ File Validation").attribute(Attribute::Bold), + )?])); + output.push(Line::default()); + + let with_issues = validations + .iter() + .filter(|validation| validation.has_issues()) + .count(); + let files = pluralize("file", validations.len() as isize, true); + let summary = if with_issues > 0 { + format!("❕ {files} found, {with_issues} with issues") + } else { + format!("βœ… {files} found, all fully correct") + }; + output.push(Line::from_iter([Span::new_styled( + summary.attribute(Attribute::Bold), + )?])); + output.push(Line::default()); + + for validation in limits.limit_files(validations.iter()) { + // Files that couldn't be parsed have never been listed; listing them is a visible change. + if validation.parse_error.is_some() { + continue; + } + let file_name = &validation.file; + let mut lines: Vec = vec![]; + match (validation.warnings.is_empty(), validation.errors.is_empty()) { + (false, false) => { + lines.push(Line::from_iter([ + Span::new_unstyled("❌ ")?, + Span::new_styled_lossy( + format!("{file_name} Has Errors And Warnings").attribute(Attribute::Bold), + ), + ])); + lines.push(Line::from_iter([ + Span::new_unstyled(" β†ͺ ❌ ")?, + Span::new_styled(String::from("Errors").attribute(Attribute::Bold))?, + ])); + for error in limits.limit_issues(validation.errors.iter()) { + lines.push(Line::from_iter([ + Span::new_unstyled(" β†ͺ ")?, + Span::new_unstyled_lossy(error), + ])); + } + lines.push(Line::from_iter([ + Span::new_unstyled(" β†ͺ ⚠️ ")?, + Span::new_styled(String::from("Warnings").attribute(Attribute::Bold))?, + ])); + for warning in limits.limit_issues(validation.warnings.iter()) { + lines.push(Line::from_iter([ + Span::new_unstyled(" β†ͺ ")?, + Span::new_unstyled_lossy(warning), + ])); + } + } + (true, false) => { + lines.push(Line::from_iter([ + Span::new_unstyled("❌ ")?, + Span::new_styled_lossy( + format!("{file_name} Has Errors").attribute(Attribute::Bold), + ), + ])); + for error in limits.limit_issues(validation.errors.iter()) { + lines.push(Line::from_iter([ + Span::new_unstyled(" β†ͺ ")?, + Span::new_unstyled_lossy(error), + ])); + } + } + (false, true) => { + lines.push(Line::from_iter([ + Span::new_unstyled("⚠️ ")?, + Span::new_styled_lossy( + format!("{file_name} Has Warnings").attribute(Attribute::Bold), + ), + ])); + for warning in limits.limit_issues(validation.warnings.iter()) { + lines.push(Line::from_iter([ + Span::new_unstyled(" β†ͺ ")?, + Span::new_unstyled_lossy(warning), + ])); + } + } + (true, true) => {} + } + let mut lines = Lines::from_iter(lines); + lines.pad_lines_left(2); + output.extend(lines); + } + if let Some(extra_files) = limits.num_exceeding_files_limit(validations.len()) { + output.push(Line::from_iter([Span::new_unstyled(format!( + "…and {} more", + extra_files + ))?])); + } + + Ok(output) +} + +/// Plain text unless superconsole is drawing to an interactive terminal: log viewers such as +/// GitHub Actions don't support OSC 8 and drop the link text along with the escape. +fn hyperlink_span(url: String, attribute: Attribute) -> anyhow::Result { + url_span(url, attribute, SuperConsole::compatible()) +} + +fn url_span(url: String, attribute: Attribute, clickable: bool) -> anyhow::Result { + let span = Span::new_styled(style(url.clone()).attribute(attribute))?; + Ok(if clickable { + span.with_hyperlink(Some(Hyperlink::new(url))) + } else { + span + }) +} + +#[cfg(test)] +mod tests { + use superconsole::{Line, style::Attribute}; + + use super::url_span; + + const URL: &str = "https://app.trunk.io/my-org/flaky-tests/test/1"; + + fn rendered(clickable: bool) -> String { + Line::from_iter([url_span(URL.into(), Attribute::Underlined, clickable).unwrap()]).render() + } + + #[test] + fn url_span_adds_osc8_only_when_clickable() { + assert!(rendered(true).contains("\x1b]8;;")); + let plain = rendered(false); + assert!(!plain.contains("\x1b]8;;")); + assert!(plain.contains(URL)); + } +} diff --git a/cli/src/report/upload.rs b/cli/src/report/upload.rs new file mode 100644 index 00000000..d518be4e --- /dev/null +++ b/cli/src/report/upload.rs @@ -0,0 +1,271 @@ +use std::collections::BTreeMap; + +use api::urls::{url_for_test_case, url_for_upload}; +use bundle::Test; +use constants::EXIT_SUCCESS; +use context::junit::validator::JunitValidationLevel; +use serde::Serialize; + +use super::{ErrorSummary, Notice}; +use crate::{upload_command::UploadRunResult, validate_command::JunitReportValidations}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UploadCommand { + Upload, + Test, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UploadReport { + pub error: Option, + pub bundle_upload_id: Option, + pub upload_url: Option, + pub validations: Vec, + pub counts: TestCounts, + pub quarantine_lookup_failed: bool, + pub quarantined: Vec, + pub failures: Vec, + pub outcome: Outcome, + pub exit_code: i32, + pub notices: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TestCounts { + pub total: usize, + pub passed: usize, + pub failed: usize, + pub quarantined: usize, + pub pass_ratio: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FailureGroup { + pub key: GroupKey, + pub tests: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum GroupKey { + File(String), + Suite(String), + Other, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FailedTest { + pub name: String, + pub suite: String, + pub class_name: Option, + pub file: Option, + pub id: String, + pub variant: Option, + pub failure_message: Option, + pub quarantined: bool, + pub link: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FileValidation { + pub file: String, + pub parse_error: Option, + pub errors: Vec, + pub warnings: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Outcome { + NoTestsFound, + AllFailuresQuarantined, + NoFailures, + NoFailuresButNonZeroExitCode, + FailuresNotQuarantined, +} + +impl UploadReport { + /// `test` exits with the test command's exit code even when the upload fails, so the + /// command decides whether an error's exit code wins. + pub fn new(result: &UploadRunResult, command: UploadCommand) -> Self { + let quarantine_context = &result.quarantine_context; + let quarantined = &quarantine_context.quarantine_status.quarantine_results; + let failures = &quarantine_context.failures; + let counts = TestCounts::new( + result.meta.junit_props.num_tests, + failures.len(), + quarantined.len(), + ); + let error = result.error_report.as_ref().map(ErrorSummary::new); + let exit_code = match command { + UploadCommand::Upload => error + .as_ref() + .and_then(|error| error.exit_code) + .unwrap_or(quarantine_context.exit_code), + UploadCommand::Test => quarantine_context.exit_code, + }; + let bundle_upload_id = &result.meta.base_props.bundle_upload_id; + + Self { + error, + bundle_upload_id: Some(bundle_upload_id.clone()).filter(|id| !id.is_empty()), + upload_url: upload_url(result), + validations: file_validations(&result.validations), + counts, + quarantine_lookup_failed: quarantine_context.fetch_status.is_failure(), + quarantined: group(quarantined, |test| failed_test(result, test, true)), + failures: group(failures, |test| failed_test(result, test, false)), + outcome: Outcome::new(&counts, quarantine_context.exit_code), + exit_code, + notices: result.notices.clone(), + } + } +} + +impl TestCounts { + fn new(total: usize, failed: usize, quarantined: usize) -> Self { + let passed = total.saturating_sub(quarantined + failed); + Self { + total, + passed, + failed, + quarantined, + pass_ratio: (total > 0).then(|| passed as f64 / total as f64), + } + } +} + +impl FileValidation { + pub fn has_issues(&self) -> bool { + self.parse_error.is_some() || !self.errors.is_empty() || !self.warnings.is_empty() + } +} + +impl Outcome { + fn new(counts: &TestCounts, test_run_exit_code: i32) -> Self { + if counts.total == 0 { + Self::NoTestsFound + } else if counts.failed == 0 && counts.quarantined > 0 { + Self::AllFailuresQuarantined + } else if counts.failed == 0 && test_run_exit_code == EXIT_SUCCESS { + Self::NoFailures + } else if counts.failed == 0 { + Self::NoFailuresButNonZeroExitCode + } else { + Self::FailuresNotQuarantined + } + } +} + +/// Link to this upload's page, when the run is in test-collection link mode. Gated on +/// `test_collection_short_id` rather than `base_props.test_collection.short_id` so an +/// exported-but-blank collection id can't yield a malformed `/collections//uploads/` +/// link. The payload supplies both halves of the key the canonical route is keyed on. +fn upload_url(result: &UploadRunResult) -> Option { + if result.hide_test_collection_links { + return None; + } + let short_id = result.test_collection_short_id.as_deref()?; + let test_collection = result.meta.base_props.test_collection.as_ref()?; + url_for_upload( + &result.api_address, + &result.meta.base_props.org, + short_id, + &test_collection.bundle_meta_id, + &test_collection.bundle_meta_created_at, + ) + .ok() +} + +fn failed_test(result: &UploadRunResult, test: &Test, quarantined: bool) -> FailedTest { + let show_collection_links = !result.hide_test_collection_links; + let link = url_for_test_case( + &result.api_address, + &result.quarantine_context.org_url_slug, + &result.quarantine_context.repo, + test, + result + .test_collection_short_id + .as_deref() + .filter(|_| show_collection_links), + result.guid_scope.as_ref().filter(|_| show_collection_links), + ); + FailedTest { + name: test.name.clone(), + suite: test.parent_name.clone(), + class_name: test.class_name.clone(), + file: test.file.clone(), + id: test.id.clone(), + variant: test.variant.clone(), + failure_message: test.failure_message.clone(), + quarantined, + link: link.ok(), + } +} + +fn group(tests: &[Test], to_failed_test: impl Fn(&Test) -> FailedTest) -> Vec { + let mut by_file: BTreeMap<&str, Vec> = BTreeMap::new(); + let mut by_suite: BTreeMap<&str, Vec> = BTreeMap::new(); + let mut other = Vec::new(); + for test in tests { + if let Some(file) = &test.file { + by_file.entry(file).or_default().push(to_failed_test(test)); + } else if !test.parent_name.is_empty() { + by_suite + .entry(&test.parent_name) + .or_default() + .push(to_failed_test(test)); + } else { + other.push(to_failed_test(test)); + } + } + let files = by_file.into_iter().map(|(file, tests)| FailureGroup { + key: GroupKey::File(file.to_string()), + tests, + }); + let suites = by_suite.into_iter().map(|(suite, tests)| FailureGroup { + key: GroupKey::Suite(suite.to_string()), + tests, + }); + let other = (!other.is_empty()).then_some(FailureGroup { + key: GroupKey::Other, + tests: other, + }); + files.chain(suites).chain(other).collect() +} + +fn file_validations(validations: &JunitReportValidations) -> Vec { + validations + .validations + .iter() + .map(|(file, validation)| match validation { + Err(error) => FileValidation { + file: file.clone(), + parse_error: Some(format!("{error:#}")), + errors: Vec::new(), + warnings: Vec::new(), + }, + Ok(validation) => { + let issues = validation.all_issues_flat(); + let messages = |level: JunitValidationLevel| { + issues + .iter() + .filter(|issue| issue.level == level) + .map(|issue| issue.error_message.clone()) + .collect() + }; + FileValidation { + file: file.clone(), + parse_error: None, + errors: messages(JunitValidationLevel::Invalid), + warnings: messages(JunitValidationLevel::SubOptimal), + } + } + }) + .collect() +} diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index 34425fc4..855ea500 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -5,19 +5,14 @@ use std::sync::mpsc::Sender; use api::client::{ApiClient, ApiErrorEndpoint}; use api::message::{GetUploadApiVersionRequest, UploadApiCaller, UploadApiVersion}; -use api::urls::{TestCaseGuidScope, url_for_test_case, url_for_upload}; -use bundle::{BundleMeta, BundlerUtil, QuarantineResolutionMode, Test, unzip_tarball}; +use api::urls::TestCaseGuidScope; +use bundle::{BundleMeta, BundlerUtil, QuarantineResolutionMode, unzip_tarball}; use clap::{ArgAction, Args}; use codeowners::OwnersSource; use constants::EXIT_SUCCESS; use context::bazel_bep::common::BepParseResult; use context::env::parser::CIPlatform; -use display::{end_output::EndOutput, message::DisplayMessage}; -use pluralizer::pluralize; -use superconsole::{ - Hyperlink, Line, Lines, Span, SuperConsole, - style::{Attribute, Color, Stylize, style}, -}; +use display::message::DisplayMessage; use tempfile::TempDir; use crate::context_quarantine::{QuarantineContext, quarantine_query_result}; @@ -29,6 +24,7 @@ use crate::{ generate_internal_file, generate_internal_file_from_bep, }, error_report::ErrorReport, + report::Notice, report_limiting::ValidationReport, test_command::TestRunResult, }; @@ -481,6 +477,7 @@ pub struct UploadRunResult { pub hide_test_collection_links: bool, pub api_address: String, pub guid_scope: Option, + pub notices: Vec, } pub struct RunUploadOptions { @@ -532,30 +529,33 @@ pub async fn run_upload( chrono::Utc::now().into() }; + let mut notices = Vec::new(); if upload_args.print_files.is_some() { - tracing::error!( - "The --print-files flag is deprecated and will be removed in a future version." - ); + let message = + "The --print-files flag is deprecated and will be removed in a future version."; + tracing::error!("{message}"); + notices.push(Notice::deprecation(message)); } if let Some(team) = &upload_args.team { if !team.is_empty() { - tracing::error!( - "The --team flag is deprecated and will be removed in a future version." - ); + let message = "The --team flag is deprecated and will be removed in a future version."; + tracing::error!("{message}"); + notices.push(Notice::deprecation(message)); } } if upload_args.hide_banner { - tracing::error!( - "The --hide-banner flag is deprecated and does nothing. It will be removed in a future version." - ); + let message = "The --hide-banner flag is deprecated and does nothing. It will be removed in a future version."; + tracing::error!("{message}"); + notices.push(Notice::deprecation(message)); } if upload_args.use_quarantining.is_some() { - tracing::warn!( - "The --use-quarantining flag is deprecated. Use --disable-quarantining instead." - ); + let message = + "The --use-quarantining flag is deprecated. Use --disable-quarantining instead."; + tracing::warn!("{message}"); + notices.push(Notice::deprecation(message)); } // Caught here rather than server-side: without a collection there is nothing for the forked @@ -615,6 +615,10 @@ pub async fn run_upload( }; api_client.set_upload_api_version(upload_api_version); + if let Some(bep_result) = &bep_result { + notices.extend(bep_result.errors.iter().map(Notice::bep_parse_error)); + } + let file_set_builder = gather_post_test_context( &mut meta, junit_path_wrappers, @@ -808,6 +812,7 @@ pub async fn run_upload( hide_test_collection_links: upload_args.hide_test_collection_links, api_address: api_client.api_host.clone(), guid_scope, + notices, }) } @@ -934,381 +939,6 @@ pub fn get_bundle_upload_id_message(bundle_upload_id: &str) -> String { format!("🏷️ Bundle Upload ID: {}", bundle_upload_id) } -/// Plain text unless superconsole is drawing to an interactive terminal: log viewers such as -/// GitHub Actions don't support OSC 8 and drop the link text along with the escape. -fn hyperlink_span(url: String, attribute: Attribute) -> anyhow::Result { - url_span(url, attribute, SuperConsole::compatible()) -} - -fn url_span(url: String, attribute: Attribute, clickable: bool) -> anyhow::Result { - let span = Span::new_styled(style(url.clone()).attribute(attribute))?; - Ok(if clickable { - span.with_hyperlink(Some(Hyperlink::new(url))) - } else { - span - }) -} - -impl UploadRunResult { - /// Link to this upload's page, when the run is in test-collection link mode. Gated on - /// `test_collection_short_id` rather than `base_props.test_collection.short_id` so an - /// exported-but-blank collection id can't yield a malformed `/collections//uploads/` - /// link. The payload supplies both halves of the key the canonical route is keyed on. - fn collection_upload_url(&self) -> Option { - if self.hide_test_collection_links { - return None; - } - let short_id = self.test_collection_short_id.as_deref()?; - let test_collection = self.meta.base_props.test_collection.as_ref()?; - url_for_upload( - &self.api_address, - &self.meta.base_props.org, - short_id, - &test_collection.bundle_meta_id, - &test_collection.bundle_meta_created_at, - ) - .ok() - } -} - -impl EndOutput for UploadRunResult { - fn output(&self) -> anyhow::Result> { - let mut output: Vec = Vec::new(); - // If there is an error report, we display it instead - if let Some(error_report) = self.error_report.as_ref() { - output.push(Line::default()); - output.extend(error_report.output()?); - if error_report.should_block_quarantining() { - return Ok(output); - } else { - output.push(Line::from_iter([Span::new_unstyled( - "Proceeding with test report...", - )?])); - output.push(Line::default()); - } - } - - // Add the bundle upload message: the short link when the run is in - // test-collection link mode, otherwise the bare id as before. - { - let bundle_upload_id = &self.meta.base_props.bundle_upload_id; - let bold = |text: String| Span::new_styled(style(text).attribute(Attribute::Bold)); - let line = match self.collection_upload_url() { - Some(url) => Some(Line::from_iter([ - bold(String::from("🏷️ Upload: "))?, - hyperlink_span(url, Attribute::Bold)?, - ])), - None if !bundle_upload_id.is_empty() => Some(Line::from_iter([bold( - get_bundle_upload_id_message(bundle_upload_id), - )?])), - None => None, - }; - if let Some(line) = line { - output.push(line); - output.push(Line::default()); - } - } - - if !self.validations.validations.is_empty() { - output.extend( - self.validations - .output_with_report_limits(&self.validation_report)?, - ); - output.push(Line::default()); - } - // Summary statistics - let total_tests = self.meta.junit_props.num_tests; - let quarantined = self - .quarantine_context - .quarantine_status - .quarantine_results - .len(); - let failures = self.quarantine_context.failures.len(); - let passes = total_tests.saturating_sub(quarantined + failures); - let pass_ratio = if total_tests > 0 { - format!("{:.1}%", (passes as f64 / total_tests as f64) * 100.0) - } else { - "N/A".to_string() - }; - output.push(Line::from_iter([Span::new_styled( - style("πŸ“š Test Report".to_string()).attribute(Attribute::Bold), - )?])); - output.push(Line::from_iter([Span::new_unstyled(format!( - " Total: {} Pass: {} Fail: {} Quarantined: {} Pass Ratio: {}", - total_tests, passes, failures, quarantined, pass_ratio - ))?])); - if self.quarantine_context.fetch_status.is_failure() { - output.push(Line::from_iter([Span::new_styled( - style(" We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures".to_string()).with(Color::Yellow).attribute(Attribute::Bold), - )?])); - } - output.push(Line::default()); - let qc = &self.quarantine_context; - let quarantined = &qc.quarantine_status.quarantine_results; - let failures = &qc.failures; - let quarantined_count = quarantined.len(); - let non_quarantined_count = failures.len(); - let all_quarantined = non_quarantined_count == 0 && quarantined_count > 0; - - let test_collection_short_id = self - .test_collection_short_id - .as_deref() - .filter(|_| !self.hide_test_collection_links); - let guid_scope = self - .guid_scope - .as_ref() - .filter(|_| !self.hide_test_collection_links); - - // Helper closure to render the test table - let render_test_table = |tests: &[Test]| -> anyhow::Result { - use std::collections::BTreeMap; - let mut output = Vec::new(); - // Group tests by file path, then by suite, then standalone - let mut groups: BTreeMap> = BTreeMap::new(); - let mut suite_groups: BTreeMap> = BTreeMap::new(); - let mut standalone: Vec<&Test> = Vec::new(); - for test in tests.iter() { - if let Some(file) = &test.file { - groups.entry(file.clone()).or_default().push(test); - } else if !test.parent_name.is_empty() { - suite_groups - .entry(test.parent_name.clone()) - .or_default() - .push(test); - } else { - standalone.push(test); - } - } - // Helper to render a group - let mut render_group = |header: &str, - color: Color, - group: &Vec<&Test>| - -> anyhow::Result<()> { - output.push(Line::from_iter([Span::new_styled( - style(header.to_string()) - .with(color) - .attribute(Attribute::Bold), - )?])); - for test in group.iter().take(3) { - let output_name = format!( - "{}{}{}", - test.parent_name, - if test.parent_name.is_empty() { "" } else { "/" }, - test.name - ); - let mut test_line = Line::from_iter([Span::new_styled_lossy( - style(output_name.to_string()).attribute(Attribute::Bold), - )]); - test_line.pad_left(2); - output.push(test_line); - let link = url_for_test_case( - &self.api_address, - &self.quarantine_context.org_url_slug, - &self.quarantine_context.repo, - test, - test_collection_short_id, - guid_scope, - )?; - let mut link_output = Line::from_iter([ - Span::new_unstyled("β€· ")?, - hyperlink_span(link, Attribute::Underlined)?, - ]); - link_output.pad_left(4); - output.push(link_output); - // Display failure message if present and enabled - // TODO: show_failure_messages is a temporary flag to show failure messages - // in the output. It should be removed once we are confident in this flow - // and we should use the validation report flag instead. - if self.show_failure_messages && test.failure_message.is_some() { - let failure_message = test.failure_message.as_ref().unwrap(); - let lines: Vec<&str> = failure_message.split('\n').collect(); - let max_lines = 20; - let shown_lines = lines.iter().take(max_lines); - let mut failure_header = Line::from_iter([Span::new_styled( - style("Failure: ".to_string()).with(Color::DarkGrey), - )?]); - failure_header.pad_left(4); - output.push(failure_header); - for (j, line) in shown_lines.enumerate() { - let sanitized_line = line.replace('\t', " ").replace('\r', ""); - if !sanitized_line.trim().is_empty() || j == 0 { - let mut failure_output = Line::from_iter([ - Span::new_unstyled(" ")?, - Span::new_styled_lossy( - style(sanitized_line.to_string()) - .with(Color::Grey) - .attribute(Attribute::Italic), - ), - ]); - failure_output.pad_left(4); - output.push(failure_output); - } - } - if lines.len() > max_lines { - let omitted = lines.len() - max_lines; - let mut more_output = Line::from_iter([ - Span::new_unstyled(" ")?, - Span::new_styled( - style(format!("…and {omitted} more lines not shown")) - .with(Color::Grey) - .attribute(Attribute::Italic), - )?, - ]); - more_output.pad_left(4); - output.push(more_output); - } - } - } - if group.len() > 3 { - let mut more_failures = Line::from_iter([Span::new_unstyled(format!( - "…and {} more failures in this group", - group.len() - 3 - ))?]); - more_failures.pad_left(2); - output.push(more_failures); - } - output.push(Line::default()); - Ok(()) - }; - // Render file path groups (cyan) - for (file, group) in &groups { - render_group(&format!("πŸ“ {}", file), Color::Cyan, group)?; - } - // Render suite groups (magenta) - for (suite, group) in &suite_groups { - render_group(&format!("πŸ“¦ {}", suite), Color::Magenta, group)?; - } - // Render standalone (yellow) - if !standalone.is_empty() { - render_group("πŸ” Other", Color::Yellow, &standalone)?; - } - let mut output = Lines(output); - output.pad_lines_left(2); - output.pad_lines_bottom(1); - Ok(output) - }; - - // Quarantined section - if quarantined_count > 0 { - output.extend(vec![ - Line::from_iter([ - Span::new_unstyled("β€οΈβ€πŸ©Ή ")?, - Span::new_styled( - style(format!( - "{} ", - pluralize("test", quarantined_count as isize, true) - )) - .attribute(Attribute::Bold), - )?, - Span::new_styled(style(format!( - "failed and {} ", - pluralize("was", quarantined_count as isize, false) - )))?, - Span::new_styled( - style(String::from("quarantined")) - .with(Color::Yellow) - .attribute(Attribute::Bold), - )?, - ]), - Line::default(), - ]); - output.extend(render_test_table(quarantined)?); - } - - // Non-quarantined failures section - if non_quarantined_count > 0 { - output.extend(vec![ - Line::from_iter([ - Span::new_unstyled("❌ ")?, - Span::new_unstyled(format!( - "{} ", - pluralize("test", non_quarantined_count as isize, true) - ))?, - Span::new_styled( - style(String::from("failed ")) - .with(Color::Red) - .attribute(Attribute::Bold), - )?, - Span::new_styled(style(format!( - "and {} ", - pluralize("was", non_quarantined_count as isize, false) - )))?, - Span::new_styled(style(String::from("not ")).attribute(Attribute::Bold))?, - Span::new_unstyled("quarantined")?, - ]), - Line::default(), - ]); - output.extend(render_test_table(failures)?); - } - - // Final messages - if self.meta.junit_props.num_tests == 0 { - output.push(Line::from_iter([Span::new_unstyled( - "⚠️ No tests were found in the provided test results", - )?])); - } else if all_quarantined { - output.push(Line::from_iter([ - Span::new_unstyled("πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success ")?, - Span::new_styled(style(format!( - "({})", - EXIT_SUCCESS - )).attribute(Attribute::Bold))? - ])); - } else if failures.is_empty() && self.quarantine_context.exit_code == EXIT_SUCCESS { - output.push(Line::from_iter([Span::new_unstyled( - "πŸŽ‰ No test failures found!", - )?])); - } else if failures.is_empty() && self.quarantine_context.exit_code != EXIT_SUCCESS { - // no test failures found but exit code not 0 - output.push(Line::from_iter([ - Span::new_unstyled( - "⚠️ No test failures found, but non zero exit code provided: ", - )?, - Span::new_styled( - style(format!("{}", self.quarantine_context.exit_code)) - .attribute(Attribute::Bold), - )?, - ])); - output.push( - Line::from_iter([ - Span::new_unstyled("This may indicate that some tests were not run or that there were other issues during the test run.")?, - ])); - } else { - output.push(Line::from_iter([ - Span::new_unstyled( - "⚠️ Some test failures were not quarantined, using exit code: ".to_string(), - )?, - Span::new_styled( - style(format!("{}", self.quarantine_context.exit_code)) - .attribute(Attribute::Bold), - )?, - ])); - } - Ok(output) - } -} - -#[cfg(test)] -mod tests { - use superconsole::{Line, style::Attribute}; - - use super::url_span; - - const URL: &str = "https://app.trunk.io/my-org/flaky-tests/test/1"; - - fn rendered(clickable: bool) -> String { - Line::from_iter([url_span(URL.into(), Attribute::Underlined, clickable).unwrap()]).render() - } - - #[test] - fn url_span_adds_osc8_only_when_clickable() { - assert!(rendered(true).contains("\x1b]8;;")); - let plain = rendered(false); - assert!(!plain.contains("\x1b]8;;")); - assert!(plain.contains(URL)); - } -} - #[cfg(test)] mod fork_pr_tests { use std::collections::HashMap; diff --git a/cli/src/validate_command.rs b/cli/src/validate_command.rs index 0b8ad1d3..330c7991 100644 --- a/cli/src/validate_command.rs +++ b/cli/src/validate_command.rs @@ -25,13 +25,13 @@ use display::end_output::EndOutput; use pluralizer::pluralize; use quick_junit::Report; use superconsole::{ - Line, Lines, Span, + Line, Span, style::{Attribute, Color, Stylize, style}, }; use crate::{ context::fall_back_to_binary_parse, error_report::InterruptingError, - report_limiting::ValidationReport, upload_command::CodeownersType, + upload_command::CodeownersType, }; #[derive(Args, Clone, Debug)] @@ -779,180 +779,11 @@ fn gen_codeowners_issues( #[derive(Debug)] pub struct JunitReportValidations { pub validations: BTreeMap>, - files: Vec, - files_without_issues: Vec, } impl JunitReportValidations { pub fn new(validations: BTreeMap>) -> Self { - let mut files: Vec = validations.keys().cloned().collect(); - files.sort(); - let mut files_without_issues: Vec = Vec::new(); - for (file_name, validation) in validations.iter() { - if let Ok(report_validation) = validation { - if report_validation.num_invalid_issues() == 0 - && report_validation.num_suboptimal_issues() == 0 - { - files_without_issues.push(file_name.clone()); - } - } - } - Self { - validations, - files, - files_without_issues, - } - } - - pub fn output_with_report_limits( - &self, - limits: &ValidationReport, - ) -> anyhow::Result> { - let mut output: Vec = Vec::new(); - if limits == &ValidationReport::None { - return Ok(output); - } - - output.push(Line::from_iter([Span::new_styled( - String::from("πŸ“‚ File Validation").attribute(Attribute::Bold), - )?])); - output.push(Line::default()); - - if self.files.is_empty() { - output.push(Line::from_iter([Span::new_styled( - "⚠️ No files found".to_string().attribute(Attribute::Bold), - )?])); - return Ok(output); - } else if self.files_without_issues.len() != self.files.len() { - // found x number of files with issues - output.push(Line::from_iter([Span::new_styled( - format!( - "❕ {} found, {} with issues", - pluralize("file", self.files.len() as isize, true), - self.files.len() - self.files_without_issues.len(), - ) - .attribute(Attribute::Bold), - )?])); - } else { - // all files are perfect - output.push(Line::from_iter([Span::new_styled( - format!( - "βœ… {} found, all fully correct", - pluralize("file", self.files.len() as isize, true) - ) - .attribute(Attribute::Bold), - )?])); - } - output.push(Line::default()); - - for (file_name, validation_reports) in limits.limit_files(self.validations.iter()) { - let mut lines: Vec = vec![]; - match validation_reports { - Err(e) => { - lines.extend([ - Line::from_iter([ - Span::new_unstyled("❌ ")?, - Span::new_styled_lossy( - format!("{file_name} Could Not Be Parsed") - .attribute(Attribute::Bold), - ), - ]), - Line::from_iter([ - Span::new_unstyled(" β†ͺ ")?, - Span::new_unstyled_lossy(format!("{:?}", e)), - ]), - ]); - } - Ok(report) => { - let issues = report.all_issues_flat(); - let sub_optimal_issues: Vec<&JunitReportValidationFlatIssue> = issues - .iter() - .filter(|issue| issue.level == JunitValidationLevel::SubOptimal) - .collect(); - let invalid_issues: Vec<&JunitReportValidationFlatIssue> = issues - .iter() - .filter(|issue| issue.level == JunitValidationLevel::Invalid) - .collect(); - match (sub_optimal_issues.is_empty(), invalid_issues.is_empty()) { - (false, false) => { - lines.push(Line::from_iter([ - Span::new_unstyled("❌ ")?, - Span::new_styled_lossy( - format!("{file_name} Has Errors And Warnings") - .attribute(Attribute::Bold), - ), - ])); - lines.push(Line::from_iter([ - Span::new_unstyled(" β†ͺ ❌ ")?, - Span::new_styled( - String::from("Errors").attribute(Attribute::Bold), - )?, - ])); - for error in limits.limit_issues(invalid_issues.iter()) { - lines.push(Line::from_iter([ - Span::new_unstyled(" β†ͺ ")?, - Span::new_unstyled_lossy(error.error_message.clone()), - ])); - } - lines.push(Line::from_iter([ - Span::new_unstyled(" β†ͺ ⚠️ ")?, - Span::new_styled( - String::from("Warnings").attribute(Attribute::Bold), - )?, - ])); - for warning in limits.limit_issues(sub_optimal_issues.iter()) { - lines.push(Line::from_iter([ - Span::new_unstyled(" β†ͺ ")?, - Span::new_unstyled_lossy(warning.error_message.clone()), - ])); - } - } - (true, false) => { - lines.push(Line::from_iter([ - Span::new_unstyled("❌ ")?, - Span::new_styled_lossy( - format!("{file_name} Has Errors").attribute(Attribute::Bold), - ), - ])); - for issue in limits.limit_issues(invalid_issues.iter()) { - lines.push(Line::from_iter([ - Span::new_unstyled(" β†ͺ ")?, - Span::new_unstyled_lossy(issue.error_message.clone()), - ])); - } - } - (false, true) => { - lines.push(Line::from_iter([ - Span::new_unstyled("⚠️ ")?, - Span::new_styled_lossy( - format!("{file_name} Has Warnings").attribute(Attribute::Bold), - ), - ])); - for warning in limits.limit_issues(sub_optimal_issues.iter()) { - lines.push(Line::from_iter([ - Span::new_unstyled(" β†ͺ ")?, - Span::new_unstyled_lossy(warning.error_message.clone()), - ])); - } - } - (true, true) => { - // pass - } - } - let mut output_lines = Lines::from_iter(lines); - output_lines.pad_lines_left(2); - output.extend(output_lines); - } - } - } - if let Some(extra_files) = limits.num_exceeding_files_limit(self.validations.len()) { - output.push(Line::from_iter([Span::new_unstyled(format!( - "…and {} more", - extra_files - ))?])); - } - - Ok(output) + Self { validations } } } diff --git a/cli/tests/summary/error_report.rs b/cli/tests/summary/error_report.rs index d946938b..be63709e 100644 --- a/cli/tests/summary/error_report.rs +++ b/cli/tests/summary/error_report.rs @@ -1,14 +1,15 @@ use api::client::ClientErrorExplanation; -use display::end_output::EndOutput; use pretty_assertions::assert_eq; use superconsole::Line; -use trunk_analytics_cli::error_report::{ErrorReport, InterruptingError}; +use trunk_analytics_cli::{ + error_report::{ErrorReport, InterruptingError}, + report::{ErrorSummary, text}, +}; use crate::unauthorized_error; fn error_summary(report: &ErrorReport) -> Vec { - report - .output() + text::render_error(&ErrorSummary::new(report)) .unwrap() .iter() .map(Line::to_unstyled) diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index 8b5c908d..340e011b 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -13,12 +13,15 @@ use context::{ }, repo::{BundleRepo, RepoUrlParts}, }; -use display::end_output::EndOutput; use pretty_assertions::assert_eq; use superconsole::Line; use trunk_analytics_cli::{ context_quarantine::{QuarantineContext, QuarantineFetchStatus}, error_report::ErrorReport, + report::{ + UploadCommand, UploadReport, + text::{self, TextOptions}, + }, report_limiting::ValidationReport, upload_command::UploadRunResult, validate_command::JunitReportValidations, @@ -31,8 +34,8 @@ const UPLOAD_ERROR_MESSAGE: &str = "There was an unexpected error that occurred while uploading test results"; fn upload_summary(result: &UploadRunResult) -> Vec { - result - .output() + let report = UploadReport::new(result, UploadCommand::Upload); + text::render_upload(&report, &TextOptions::from(result)) .unwrap() .iter() .map(Line::to_unstyled) @@ -96,6 +99,7 @@ fn upload_result(num_tests: usize) -> UploadRunResult { hide_test_collection_links: false, api_address: String::from("https://api.trunk.io"), guid_scope: None, + notices: Vec::new(), } } @@ -815,3 +819,37 @@ fn warns_about_a_non_zero_exit_code_without_failures() { ] ); } + +#[test] +fn upload_exits_with_the_exit_code_of_an_error_but_test_does_not() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.error_report = Some(ErrorReport::new( + unauthorized_error(), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + UploadReport::new(&result, UploadCommand::Upload).exit_code, + 70 + ); + assert_eq!(UploadReport::new(&result, UploadCommand::Test).exit_code, 1); +} + +#[test] +fn exits_with_the_test_run_exit_code_when_an_error_has_none() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.error_report = Some(ErrorReport::new( + anyhow::anyhow!("HTTP status server error (500 Internal Server Error)"), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + UploadReport::new(&result, UploadCommand::Upload).exit_code, + 1 + ); + assert_eq!(UploadReport::new(&result, UploadCommand::Test).exit_code, 1); +} From 0a5c5c07e0bcf15d057fbe6645544aa7ade4d318 Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 28 Oct 2026 15:58:22 +0000 Subject: [PATCH 04/19] refactor(cli): rename the report module to summary_output "report" already means the RSpec test report and JUnit reports in this repo. Co-Authored-By: Claude Opus 5.5 --- cli/src/lib.rs | 2 +- cli/src/main.rs | 2 +- cli/src/{report => summary_output}/error.rs | 0 cli/src/{report => summary_output}/mod.rs | 0 cli/src/{report => summary_output}/notice.rs | 0 cli/src/{report => summary_output}/text.rs | 0 cli/src/{report => summary_output}/upload.rs | 0 cli/src/upload_command.rs | 2 +- cli/tests/summary/error_report.rs | 2 +- cli/tests/summary/upload.rs | 4 ++-- 10 files changed, 6 insertions(+), 6 deletions(-) rename cli/src/{report => summary_output}/error.rs (100%) rename cli/src/{report => summary_output}/mod.rs (100%) rename cli/src/{report => summary_output}/notice.rs (100%) rename cli/src/{report => summary_output}/text.rs (100%) rename cli/src/{report => summary_output}/upload.rs (100%) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index e7e10ab9..fbc108c6 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -2,8 +2,8 @@ pub mod context; pub mod context_quarantine; pub mod error_report; pub mod print; -pub mod report; pub mod report_limiting; +pub mod summary_output; pub mod test_command; pub mod upload_command; pub mod validate_command; diff --git a/cli/src/main.rs b/cli/src/main.rs index e70e2aec..bcee75e8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -17,7 +17,7 @@ use tracing_subscriber::{filter::FilterFn, prelude::*}; use trunk_analytics_cli::{ context::gather_debug_props, error_report::ErrorReport, - report::{ + summary_output::{ ErrorSummary, UploadCommand, UploadReport, text::{TextOptions, UploadText}, }, diff --git a/cli/src/report/error.rs b/cli/src/summary_output/error.rs similarity index 100% rename from cli/src/report/error.rs rename to cli/src/summary_output/error.rs diff --git a/cli/src/report/mod.rs b/cli/src/summary_output/mod.rs similarity index 100% rename from cli/src/report/mod.rs rename to cli/src/summary_output/mod.rs diff --git a/cli/src/report/notice.rs b/cli/src/summary_output/notice.rs similarity index 100% rename from cli/src/report/notice.rs rename to cli/src/summary_output/notice.rs diff --git a/cli/src/report/text.rs b/cli/src/summary_output/text.rs similarity index 100% rename from cli/src/report/text.rs rename to cli/src/summary_output/text.rs diff --git a/cli/src/report/upload.rs b/cli/src/summary_output/upload.rs similarity index 100% rename from cli/src/report/upload.rs rename to cli/src/summary_output/upload.rs diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index 855ea500..48861e7a 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -24,8 +24,8 @@ use crate::{ generate_internal_file, generate_internal_file_from_bep, }, error_report::ErrorReport, - report::Notice, report_limiting::ValidationReport, + summary_output::Notice, test_command::TestRunResult, }; diff --git a/cli/tests/summary/error_report.rs b/cli/tests/summary/error_report.rs index be63709e..f916e2fb 100644 --- a/cli/tests/summary/error_report.rs +++ b/cli/tests/summary/error_report.rs @@ -3,7 +3,7 @@ use pretty_assertions::assert_eq; use superconsole::Line; use trunk_analytics_cli::{ error_report::{ErrorReport, InterruptingError}, - report::{ErrorSummary, text}, + summary_output::{ErrorSummary, text}, }; use crate::unauthorized_error; diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index 340e011b..f772b68e 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -18,11 +18,11 @@ use superconsole::Line; use trunk_analytics_cli::{ context_quarantine::{QuarantineContext, QuarantineFetchStatus}, error_report::ErrorReport, - report::{ + report_limiting::ValidationReport, + summary_output::{ UploadCommand, UploadReport, text::{self, TextOptions}, }, - report_limiting::ValidationReport, upload_command::UploadRunResult, validate_command::JunitReportValidations, }; From c103d2a9e153dff5ef519ee3d017401e9a05bd1a Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 28 Oct 2026 16:03:20 +0000 Subject: [PATCH 05/19] refactor(cli): fold failure groups, name the command, and split failure text in the upload summary - GroupKey sorts files, then suites, then the rest, so one ordered map replaces the three accumulators. - The summary records which command produced it. - A failure's message attribute and text are kept apart. bundle::Test still uploads the text-else-message as failure_message; the split fields are serde(skip) so the bundle payload doesn't change. Co-Authored-By: Claude Opus 5.5 --- api/src/urls.rs | 2 + bundle/src/types.rs | 11 ++++++ cli/src/context_quarantine.rs | 19 ++++++--- cli/src/summary_output/text.rs | 3 +- cli/src/summary_output/upload.rs | 67 +++++++++++++++++--------------- cli/tests/summary/upload.rs | 4 +- 6 files changed, 67 insertions(+), 39 deletions(-) diff --git a/api/src/urls.rs b/api/src/urls.rs index c314494f..eaad48b7 100644 --- a/api/src/urls.rs +++ b/api/src/urls.rs @@ -151,6 +151,8 @@ mod tests { timestamp_millis: None, is_quarantined: false, failure_message: None, + failure_message_attribute: None, + failure_text: None, variant: None, } } diff --git a/bundle/src/types.rs b/bundle/src/types.rs index bd2d4799..de1d4ce8 100644 --- a/bundle/src/types.rs +++ b/bundle/src/types.rs @@ -23,7 +23,14 @@ pub struct Test { /// Added in v0.6.9 pub timestamp_millis: Option, pub is_quarantined: bool, + /// What's uploaded: the failure's text, else its `message` attribute. pub failure_message: Option, + /// The failure's `message` attribute, kept apart for the CLI's summary. Not uploaded. + #[serde(skip)] + pub failure_message_attribute: Option, + /// The failure's text, usually a stack trace, kept apart for the CLI's summary. Not uploaded. + #[serde(skip)] + pub failure_text: Option, pub variant: Option, } @@ -49,6 +56,8 @@ impl Test { timestamp_millis, is_quarantined: false, failure_message: None, + failure_message_attribute: None, + failure_text: None, variant: Some(variant.to_string()), }; @@ -198,6 +207,8 @@ mod tests { timestamp_millis: Some(0), is_quarantined: false, failure_message: None, + failure_message_attribute: None, + failure_text: None, variant: Some(String::new()), }; assert_eq!(result.name, name); diff --git a/cli/src/context_quarantine.rs b/cli/src/context_quarantine.rs index dc4d3816..d3f5cab1 100644 --- a/cli/src/context_quarantine.rs +++ b/cli/src/context_quarantine.rs @@ -112,13 +112,13 @@ fn convert_case_to_test + ToString>( .or(suite.timestamp_micros) .unwrap_or(0), ))); - let failure_message = match &case.status.non_success { - Some(non_success) => non_success - .description - .clone() - .or_else(|| non_success.message.clone()), - _ => None, + let (failure_message_attribute, failure_text) = match &case.status.non_success { + Some(non_success) => (non_success.message.clone(), non_success.description.clone()), + None => (None, None), }; + let failure_message = failure_text + .clone() + .or_else(|| failure_message_attribute.clone()); let mut test = Test { name, parent_name, @@ -128,6 +128,8 @@ fn convert_case_to_test + ToString>( timestamp_millis, is_quarantined: case.is_quarantined(), failure_message, + failure_message_attribute, + failure_text, variant: Some(variant.to_string()), }; if let Some(id) = case.extra().get("id") { @@ -781,6 +783,11 @@ mod tests { assert_eq!(test.name, "test_case"); assert_eq!(test.failure_message, Some("This is a failure".to_string())); + assert_eq!( + test.failure_message_attribute, + Some("Failure message".to_string()) + ); + assert_eq!(test.failure_text, Some("This is a failure".to_string())); } #[test] diff --git a/cli/src/summary_output/text.rs b/cli/src/summary_output/text.rs index 19f867b9..7355e583 100644 --- a/cli/src/summary_output/text.rs +++ b/cli/src/summary_output/text.rs @@ -297,8 +297,9 @@ fn render_groups(groups: &[FailureGroup], options: &TextOptions) -> anyhow::Resu link_output.pad_left(4); output.push(link_output); if let Some(failure_message) = test - .failure_message + .failure_text .as_ref() + .or(test.failure_message.as_ref()) .filter(|_| options.show_failure_messages) { output.extend(render_failure_message(failure_message)?); diff --git a/cli/src/summary_output/upload.rs b/cli/src/summary_output/upload.rs index d518be4e..1af2aae0 100644 --- a/cli/src/summary_output/upload.rs +++ b/cli/src/summary_output/upload.rs @@ -9,7 +9,8 @@ use serde::Serialize; use super::{ErrorSummary, Notice}; use crate::{upload_command::UploadRunResult, validate_command::JunitReportValidations}; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] pub enum UploadCommand { Upload, Test, @@ -18,6 +19,7 @@ pub enum UploadCommand { #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct UploadReport { + pub command: UploadCommand, pub error: Option, pub bundle_upload_id: Option, pub upload_url: Option, @@ -48,7 +50,8 @@ pub struct FailureGroup { pub tests: Vec, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +/// Groups sort by file, then by suite, then the rest, in declaration order. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)] #[serde(rename_all = "camelCase")] pub enum GroupKey { File(String), @@ -65,7 +68,10 @@ pub struct FailedTest { pub file: Option, pub id: String, pub variant: Option, + /// The failure's `message` attribute. pub failure_message: Option, + /// The failure's text, usually a stack trace. + pub failure_text: Option, pub quarantined: bool, pub link: Option, } @@ -112,6 +118,7 @@ impl UploadReport { let bundle_upload_id = &result.meta.base_props.bundle_upload_id; Self { + command, error, bundle_upload_id: Some(bundle_upload_id.clone()).filter(|id| !id.is_empty()), upload_url: upload_url(result), @@ -127,6 +134,16 @@ impl UploadReport { } } +impl GroupKey { + fn of(test: &Test) -> Self { + match &test.file { + Some(file) => Self::File(file.clone()), + None if !test.parent_name.is_empty() => Self::Suite(test.parent_name.clone()), + None => Self::Other, + } + } +} + impl TestCounts { fn new(total: usize, failed: usize, quarantined: usize) -> Self { let passed = total.saturating_sub(quarantined + failed); @@ -202,41 +219,29 @@ fn failed_test(result: &UploadRunResult, test: &Test, quarantined: bool) -> Fail file: test.file.clone(), id: test.id.clone(), variant: test.variant.clone(), - failure_message: test.failure_message.clone(), + failure_message: test.failure_message_attribute.clone(), + failure_text: test.failure_text.clone(), quarantined, link: link.ok(), } } fn group(tests: &[Test], to_failed_test: impl Fn(&Test) -> FailedTest) -> Vec { - let mut by_file: BTreeMap<&str, Vec> = BTreeMap::new(); - let mut by_suite: BTreeMap<&str, Vec> = BTreeMap::new(); - let mut other = Vec::new(); - for test in tests { - if let Some(file) = &test.file { - by_file.entry(file).or_default().push(to_failed_test(test)); - } else if !test.parent_name.is_empty() { - by_suite - .entry(&test.parent_name) - .or_default() - .push(to_failed_test(test)); - } else { - other.push(to_failed_test(test)); - } - } - let files = by_file.into_iter().map(|(file, tests)| FailureGroup { - key: GroupKey::File(file.to_string()), - tests, - }); - let suites = by_suite.into_iter().map(|(suite, tests)| FailureGroup { - key: GroupKey::Suite(suite.to_string()), - tests, - }); - let other = (!other.is_empty()).then_some(FailureGroup { - key: GroupKey::Other, - tests: other, - }); - files.chain(suites).chain(other).collect() + tests + .iter() + .fold( + BTreeMap::>::new(), + |mut groups, test| { + groups + .entry(GroupKey::of(test)) + .or_default() + .push(to_failed_test(test)); + groups + }, + ) + .into_iter() + .map(|(key, tests)| FailureGroup { key, tests }) + .collect() } fn file_validations(validations: &JunitReportValidations) -> Vec { diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index f772b68e..dae9d82b 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -123,6 +123,8 @@ fn failure(name: &str, parent_name: &str, file: Option<&str>) -> Test { timestamp_millis: None, is_quarantined: false, failure_message: None, + failure_message_attribute: None, + failure_text: None, variant: None, } } @@ -562,7 +564,7 @@ fn prints_long_failure_messages_when_asked() { ]; message_lines.extend((6..=23).map(|n| format!("frame {n}"))); result.quarantine_context.failures = vec![Test { - failure_message: Some(message_lines.join("\n")), + failure_text: Some(message_lines.join("\n")), ..failure("adds", "MathTests", None) }]; From 024bef31c190a6c82e30ff5ebef5a05f4dd258ce Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 28 Oct 2026 21:00:24 +0000 Subject: [PATCH 06/19] test(api): set the split failure fields in the v2 client tests Co-Authored-By: Claude Opus 5.5 --- api/tests/v2_client.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/tests/v2_client.rs b/api/tests/v2_client.rs index 80c01095..ef9ed528 100644 --- a/api/tests/v2_client.rs +++ b/api/tests/v2_client.rs @@ -117,6 +117,8 @@ fn failure(index: usize) -> Test { timestamp_millis: None, is_quarantined: false, failure_message: None, + failure_message_attribute: None, + failure_text: None, variant: None, } } From 5c42fea893d27ec155111e21e51b5cb69312a23b Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Fri, 23 Oct 2026 22:58:04 +0000 Subject: [PATCH 07/19] refactor(cli): build the validate summary as data, separate from rendering ValidateReport, in the report module, works out everything the validate summary says once: the matched files, one FileValidation per file, the CODEOWNERS check, the counts and the exit code. main exits with report.exit_code, and the terminal output, including the EndOutput impl for ValidateRunResult, moves into the text renderer. The report derives Serialize, for the JSON output to come. validate and upload's validation section now read the same FileValidation. It gains test suite and test case counts, which are null for a file that wasn't read as exactly one report. Its parse error is now an error plus its causes, like ErrorSummary, because validate prints the error the way anyhow's {:?} does. validate collects its --hide-banner deprecation and BEP parse errors as notices, like upload. The lock-down tests pass unchanged. Refs TRUNK-22402 Co-Authored-By: Claude Opus 5.5 --- cli/src/main.rs | 9 +- cli/src/summary_output/mod.rs | 6 +- cli/src/summary_output/text.rs | 256 ++++++++++++++++++++- cli/src/summary_output/upload.rs | 26 ++- cli/src/summary_output/validate.rs | 158 +++++++++++++ cli/src/validate_command.rs | 350 +++-------------------------- 6 files changed, 472 insertions(+), 333 deletions(-) create mode 100644 cli/src/summary_output/validate.rs diff --git a/cli/src/main.rs b/cli/src/main.rs index bcee75e8..8462faf9 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -18,7 +18,7 @@ use trunk_analytics_cli::{ context::gather_debug_props, error_report::ErrorReport, summary_output::{ - ErrorSummary, UploadCommand, UploadReport, + ErrorSummary, UploadCommand, UploadReport, ValidateReport, text::{TextOptions, UploadText}, }, test_command::{TestArgs, run_test}, @@ -176,12 +176,13 @@ fn main() -> anyhow::Result<()> { close_out_and_exit(exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(RunResult::Validate(run_result))) => { - let result_ptr = Arc::new(run_result); + let report = ValidateReport::new(&run_result); + let exit_code = report.exit_code; send_message( - DisplayMessage::Final(result_ptr.clone(), String::from("validate display")), + DisplayMessage::Final(Arc::new(report), String::from("validate display")), &render_sender, ); - close_out_and_exit(result_ptr.exit_code(), guard, render_sender, render_handle) + close_out_and_exit(exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Err(error)) => { let error = ErrorSummary::new(&ErrorReport::new(error, org_url_slug, None)); diff --git a/cli/src/summary_output/mod.rs b/cli/src/summary_output/mod.rs index 3958ec5b..4de156fa 100644 --- a/cli/src/summary_output/mod.rs +++ b/cli/src/summary_output/mod.rs @@ -2,10 +2,12 @@ mod error; mod notice; pub mod text; mod upload; +mod validate; pub use error::ErrorSummary; pub use notice::{Notice, NoticeKind}; pub use upload::{ - FailedTest, FailureGroup, FileValidation, GroupKey, Outcome, TestCounts, UploadCommand, - UploadReport, + FailedTest, FailureGroup, FileValidation, GroupKey, Outcome, ParseError, TestCounts, + UploadCommand, UploadReport, }; +pub use validate::{Codeowners, MatchedFiles, ValidateReport, ValidationCounts}; diff --git a/cli/src/summary_output/text.rs b/cli/src/summary_output/text.rs index 7355e583..845ebc5d 100644 --- a/cli/src/summary_output/text.rs +++ b/cli/src/summary_output/text.rs @@ -7,12 +7,14 @@ use superconsole::{ }; use super::{ - ErrorSummary, FailureGroup, FileValidation, GroupKey, Outcome, TestCounts, UploadReport, + ErrorSummary, FailureGroup, FileValidation, GroupKey, NoticeKind, Outcome, ParseError, + TestCounts, UploadReport, ValidateReport, ValidationCounts, }; use crate::{ error_report::{CONNECTION_REFUSED_CONTEXT, ErrorKind, HELP_TEXT, UNAUTHORIZED_CONTEXT}, report_limiting::ValidationReport, upload_command::{UploadRunResult, get_bundle_upload_id_message}, + validate_command::ValidateRunResult, }; /// Flags that change how much of the summary to show, not what it says. @@ -485,11 +487,243 @@ fn url_span(url: String, attribute: Attribute, clickable: bool) -> anyhow::Resul }) } +impl EndOutput for ValidateReport { + fn output(&self) -> anyhow::Result> { + render_validate(self) + } +} + +impl EndOutput for ValidateRunResult { + fn output(&self) -> anyhow::Result> { + render_validate(&ValidateReport::new(self)) + } +} + +pub fn render_validate(report: &ValidateReport) -> anyhow::Result> { + let mut output: Vec = Vec::new(); + + let bep_parse_errors: Vec<&String> = report + .notices + .iter() + .filter(|notice| notice.kind == NoticeKind::BepParseError) + .map(|notice| ¬ice.message) + .collect(); + if !bep_parse_errors.is_empty() { + output.push(Line::from_iter([ + Span::new_unstyled("⚠️ BEP file had parse errors: ")?, + Span::new_unstyled_lossy(format!("{:?}", bep_parse_errors)), + ])); + output.push(Line::default()); + } + + let num_files: usize = report + .matched_files + .iter() + .map(|matched| matched.files.len()) + .sum(); + output.push(Line::from_iter([Span::new_unstyled(format!( + "πŸ“š Validating {} files:", + num_files + ))?])); + for matched in report.matched_files.iter() { + output.push(Line::from_iter([Span::new_unstyled(format!( + " Files matching: {}:", + matched.glob + ))?])); + for file in matched.files.iter() { + output.push(Line::from_iter([Span::new_unstyled(format!( + " {}", + file + ))?])) + } + output.push(Line::default()); + } + + // Files that weren't read as exactly one report have no test counts. + output.push(Line::from_iter([Span::new_styled( + style(String::from("File parse issues")).attribute(Attribute::Bold), + )?])); + for validation in report + .validations + .iter() + .filter(|validation| validation.test_case_count.is_none()) + { + let num_errors = validation.errors.len() + usize::from(validation.parse_error.is_some()); + output.push(file_title(validation, num_errors)?); + output.push(Line::from_iter([Span::new_styled( + style(format!( + "{} errors, {} warnings", + num_errors, + validation.warnings.len() + )) + .attribute(Attribute::Italic), + )?])); + let parse_error = validation.parse_error.as_ref().map(anyhow_debug); + output.extend(render_issues( + parse_error.iter().chain(validation.errors.iter()), + &validation.warnings, + )?); + output.push(Line::default()); + } + + output.push(Line::from_iter([Span::new_styled( + style(String::from("Test Validation Issues")).attribute(Attribute::Bold), + )?])); + for validation in report.validations.iter() { + let (Some(num_test_suites), Some(num_test_cases)) = + (validation.test_suite_count, validation.test_case_count) + else { + continue; + }; + output.push(file_title(validation, validation.errors.len())?); + output.push(Line::from_iter([Span::new_styled( + style(format!( + "{} test suites, {} test cases, {} errors, {} warnings", + num_test_suites, + num_test_cases, + validation.errors.len(), + validation.warnings.len(), + )) + .attribute(Attribute::Italic), + )?])); + output.extend(render_issues( + validation.errors.iter(), + &validation.warnings, + )?); + } + + output.push(Line::default()); + output.push(Line::from_iter([Span::new_styled( + style(String::from("Checking for codeowners file...")).attribute(Attribute::Bold), + )?])); + match &report.codeowners { + None => { + output.push(Line::from_iter([Span::new_unstyled( + " No codeowners file found", + )?])); + } + Some(codeowners) => { + output.push(Line::from_iter([Span::new_styled_lossy( + style(format!(" Found codeowners path: {:?}", codeowners.path)) + .attribute(Attribute::Italic), + )])); + for warning in codeowners.warnings.iter() { + output.push(Line::from_iter([Span::new_unstyled_lossy(format!( + " {}", + warning + ))])); + } + } + } + output.push(Line::default()); + + let ValidationCounts { + files, + valid_files, + files_with_warnings, + files_with_errors, + warnings, + errors, + } = report.counts; + output.push(Line::from_iter([ + Span::new_styled(style(format!("{}", valid_files)).with(Color::Green))?, + Span::new_unstyled(format!( + " {}, ", + pluralize("valid file", valid_files as isize, false) + ))?, + Span::new_styled(style(format!("{}", files_with_warnings)).with(Color::Yellow))?, + Span::new_unstyled(format!( + " {} with warnings, and ", + pluralize("file", files_with_warnings as isize, false) + ))?, + Span::new_styled(style(format!("{}", files_with_errors)).with(Color::Red))?, + Span::new_unstyled(format!( + " {} with errors, ", + pluralize("file", files_with_errors as isize, false) + ))?, + Span::new_unstyled(format!( + "with {} total (a file is double counted if it has both errors and warnings)", + pluralize("file", files as isize, true) + ))?, + ])); + output.push(Line::from_iter([ + Span::new_styled(style(format!("{}", warnings)).with(Color::Yellow))?, + Span::new_unstyled(format!( + " {}, and ", + pluralize("warning", warnings as isize, false) + ))?, + Span::new_styled(style(format!("{}", errors)).with(Color::Red))?, + Span::new_unstyled(format!(" {}", pluralize("error", errors as isize, false)))?, + ])); + + Ok(output) +} + +fn file_title(validation: &FileValidation, num_errors: usize) -> anyhow::Result { + let color = if num_errors > 0 { + Color::Red + } else if !validation.warnings.is_empty() { + Color::Yellow + } else { + Color::Green + }; + Ok(Line::from_iter([Span::new_styled_lossy( + style(validation.file.clone()).with(color), + )])) +} + +fn render_issues<'a>( + errors: impl Iterator, + warnings: &[String], +) -> anyhow::Result> { + let mut output = Vec::new(); + let mut errors = errors.peekable(); + if errors.peek().is_some() { + output.push(Line::from_iter([Span::new_unstyled(" ❌ Errors:")?])); + for error in errors { + output.push(Line::from_iter([Span::new_unstyled_lossy(format!( + " {}", + error + ))])); + } + } + if !warnings.is_empty() { + output.push(Line::from_iter([Span::new_unstyled(" ⚠️ Warnings:")?])); + for warning in warnings { + output.push(Line::from_iter([Span::new_unstyled_lossy(format!( + " {}", + warning + ))])); + } + } + Ok(output) +} + +/// The error as `{:?}` on an `anyhow::Error` prints it, which `validate` has always shown: +/// the error, then its causes, indented (and numbered when there's more than one). +fn anyhow_debug(parse_error: &ParseError) -> String { + let mut text = parse_error.error.clone(); + if parse_error.causes.is_empty() { + return text; + } + text.push_str("\n\nCaused by:"); + let numbered = parse_error.causes.len() > 1; + for (n, cause) in parse_error.causes.iter().enumerate() { + text.push('\n'); + let (first, rest) = if numbered { + (format!("{n: >5}: "), " ") + } else { + (String::from(" "), " ") + }; + text.push_str(&first); + text.push_str(&cause.replace('\n', &format!("\n{rest}"))); + } + text +} + #[cfg(test)] mod tests { - use superconsole::{Line, style::Attribute}; - - use super::url_span; + use super::*; const URL: &str = "https://app.trunk.io/my-org/flaky-tests/test/1"; @@ -504,4 +738,18 @@ mod tests { assert!(!plain.contains("\x1b]8;;")); assert!(plain.contains(URL)); } + + #[test] + fn anyhow_debug_matches_anyhow() { + let errors = [ + anyhow::anyhow!("no causes"), + anyhow::anyhow!("cause").context("error"), + anyhow::anyhow!("root\nspans lines") + .context("middle") + .context("error\nspans lines"), + ]; + for error in errors { + assert_eq!(anyhow_debug(&ParseError::new(&error)), format!("{error:?}")); + } + } } diff --git a/cli/src/summary_output/upload.rs b/cli/src/summary_output/upload.rs index 1af2aae0..3388af5d 100644 --- a/cli/src/summary_output/upload.rs +++ b/cli/src/summary_output/upload.rs @@ -80,9 +80,18 @@ pub struct FailedTest { #[serde(rename_all = "camelCase")] pub struct FileValidation { pub file: String, - pub parse_error: Option, + pub parse_error: Option, pub errors: Vec, pub warnings: Vec, + pub test_suite_count: Option, + pub test_case_count: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ParseError { + pub error: String, + pub causes: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] @@ -163,6 +172,15 @@ impl FileValidation { } } +impl ParseError { + pub fn new(error: &anyhow::Error) -> Self { + Self { + error: error.to_string(), + causes: error.chain().skip(1).map(ToString::to_string).collect(), + } + } +} + impl Outcome { fn new(counts: &TestCounts, test_run_exit_code: i32) -> Self { if counts.total == 0 { @@ -251,9 +269,11 @@ fn file_validations(validations: &JunitReportValidations) -> Vec .map(|(file, validation)| match validation { Err(error) => FileValidation { file: file.clone(), - parse_error: Some(format!("{error:#}")), + parse_error: Some(ParseError::new(error)), errors: Vec::new(), warnings: Vec::new(), + test_suite_count: None, + test_case_count: None, }, Ok(validation) => { let issues = validation.all_issues_flat(); @@ -269,6 +289,8 @@ fn file_validations(validations: &JunitReportValidations) -> Vec parse_error: None, errors: messages(JunitValidationLevel::Invalid), warnings: messages(JunitValidationLevel::SubOptimal), + test_suite_count: Some(validation.test_suites().len()), + test_case_count: Some(validation.test_cases().len()), } } }) diff --git a/cli/src/summary_output/validate.rs b/cli/src/summary_output/validate.rs new file mode 100644 index 00000000..171c2a00 --- /dev/null +++ b/cli/src/summary_output/validate.rs @@ -0,0 +1,158 @@ +use std::collections::HashSet; + +use constants::{EXIT_FAILURE, EXIT_SUCCESS}; +use context::junit::validator::JunitReportValidationFlatIssue; +use serde::Serialize; + +use super::{FileValidation, Notice, ParseError}; +use crate::validate_command::{ParseIssues, TestIssues, ValidateRunResult}; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ValidateReport { + pub matched_files: Vec, + pub validations: Vec, + pub codeowners: Option, + pub counts: ValidationCounts, + pub exit_code: i32, + pub notices: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MatchedFiles { + pub glob: String, + pub files: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Codeowners { + pub path: String, + pub warnings: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ValidationCounts { + pub files: usize, + pub valid_files: usize, + pub files_with_warnings: usize, + pub files_with_errors: usize, + pub warnings: usize, + pub errors: usize, +} + +impl ValidateReport { + pub fn new(result: &ValidateRunResult) -> Self { + let matched_files: Vec = result + .file_sets + .iter() + .map(|file_set| MatchedFiles { + glob: file_set.glob.clone(), + files: file_set.file_paths.clone(), + }) + .collect(); + let mut validations: Vec = result + .file_parse_issues + .iter() + .map(parse_validation) + .chain(result.test_issues.iter().map(test_validation)) + .collect(); + validations.sort_by(|a, b| a.file.cmp(&b.file)); + let has_errors = validations + .iter() + .any(|validation| validation.parse_error.is_some() || !validation.errors.is_empty()); + let mut notices = result.notices.clone(); + if let Some(bep_result) = &result.bep_result { + notices.extend(bep_result.errors.iter().map(Notice::bep_parse_error)); + } + + Self { + counts: ValidationCounts::new(&matched_files, &validations), + matched_files, + validations, + codeowners: result + .codeowners_issues + .as_ref() + .map(|codeowners| Codeowners { + path: codeowners.file_path.to_string_lossy().into_owned(), + warnings: codeowners.warnings.clone(), + }), + exit_code: if has_errors { + EXIT_FAILURE + } else { + EXIT_SUCCESS + }, + notices, + } + } +} + +impl ValidationCounts { + /// `files` counts each file once, even when several globs match it. A file with both errors + /// and warnings counts in both `files_with_errors` and `files_with_warnings`. + fn new(matched_files: &[MatchedFiles], validations: &[FileValidation]) -> Self { + let files: HashSet<&str> = matched_files + .iter() + .flat_map(|matched| matched.files.iter().map(String::as_str)) + .collect(); + let mut valid_files = files.clone(); + let mut files_with_warnings = HashSet::new(); + let mut files_with_errors = HashSet::new(); + let mut warnings = 0; + let mut errors = 0; + for validation in validations { + let file = validation.file.as_str(); + let num_errors = + validation.errors.len() + usize::from(validation.parse_error.is_some()); + if num_errors > 0 { + errors += num_errors; + files_with_errors.insert(file); + valid_files.remove(file); + } + if !validation.warnings.is_empty() { + warnings += validation.warnings.len(); + files_with_warnings.insert(file); + valid_files.remove(file); + } + } + Self { + files: files.len(), + valid_files: valid_files.len(), + files_with_warnings: files_with_warnings.len(), + files_with_errors: files_with_errors.len(), + warnings, + errors, + } + } +} + +/// A file that wasn't read as exactly one report, so it has no test counts. +fn parse_validation(issues: &ParseIssues) -> FileValidation { + FileValidation { + file: issues.file_path.clone(), + parse_error: issues.fatal_error.as_ref().map(ParseError::new), + errors: issues.errors.iter().map(ToString::to_string).collect(), + warnings: issues.warnings.iter().map(ToString::to_string).collect(), + test_suite_count: None, + test_case_count: None, + } +} + +fn test_validation(issues: &TestIssues) -> FileValidation { + let messages = |issues: &[JunitReportValidationFlatIssue]| { + issues + .iter() + .map(|issue| issue.error_message.clone()) + .collect() + }; + FileValidation { + file: issues.file_path.clone(), + parse_error: None, + errors: messages(&issues.errors), + warnings: messages(&issues.warnings), + test_suite_count: Some(issues.num_test_suites), + test_case_count: Some(issues.num_test_cases), + } +} diff --git a/cli/src/validate_command.rs b/cli/src/validate_command.rs index 330c7991..9b33fb70 100644 --- a/cli/src/validate_command.rs +++ b/cli/src/validate_command.rs @@ -1,13 +1,8 @@ -use std::{ - collections::{BTreeMap, HashSet}, - io::BufReader, - path::PathBuf, -}; +use std::{collections::BTreeMap, io::BufReader, path::PathBuf}; use bundle::{FileSet, FileSetBuilder, FileSetTestRunnerReport}; use clap::{ArgAction, Args, arg}; use codeowners::{CodeOwners, OwnersSource}; -use constants::{EXIT_FAILURE, EXIT_SUCCESS}; use context::{ bazel_bep::{common::BepParseResult, parser::BazelBepParser}, junit::{ @@ -21,16 +16,10 @@ use context::{ }, }, }; -use display::end_output::EndOutput; -use pluralizer::pluralize; use quick_junit::Report; -use superconsole::{ - Line, Span, - style::{Attribute, Color, Stylize, style}, -}; use crate::{ - context::fall_back_to_binary_parse, error_report::InterruptingError, + context::fall_back_to_binary_parse, error_report::InterruptingError, report::Notice, upload_command::CodeownersType, }; @@ -89,331 +78,46 @@ pub struct ValidateArgs { #[derive(Debug)] pub struct BepValidateResult { - errors: Vec, + pub(crate) errors: Vec, } #[derive(Debug)] pub struct FileSetResult { - glob: String, - file_paths: Vec, + pub(crate) glob: String, + pub(crate) file_paths: Vec, } #[derive(Debug)] pub struct ParseIssues { - file_path: String, - fatal_error: Option, - errors: Vec, - warnings: Vec, + pub(crate) file_path: String, + pub(crate) fatal_error: Option, + pub(crate) errors: Vec, + pub(crate) warnings: Vec, } #[derive(Debug)] pub struct TestIssues { - file_path: String, - num_test_suites: usize, - num_test_cases: usize, - errors: Vec, - warnings: Vec, + pub(crate) file_path: String, + pub(crate) num_test_suites: usize, + pub(crate) num_test_cases: usize, + pub(crate) errors: Vec, + pub(crate) warnings: Vec, } #[derive(Debug)] pub struct CodeownersIssues { - file_path: PathBuf, - warnings: Vec, + pub(crate) file_path: PathBuf, + pub(crate) warnings: Vec, } #[derive(Debug)] pub struct ValidateRunResult { - bep_result: Option, - file_sets: Vec, - file_parse_issues: Vec, - test_issues: Vec, - codeowners_issues: Option, -} - -impl ValidateRunResult { - pub fn exit_code(&self) -> i32 { - let has_errors = self.file_parse_issues.iter().any(|file_parse_issue| { - file_parse_issue.fatal_error.is_some() || !file_parse_issue.errors.is_empty() - }) || self - .test_issues - .iter() - .any(|test_issue| !test_issue.errors.is_empty()); - if has_errors { - EXIT_FAILURE - } else { - EXIT_SUCCESS - } - } - - fn num_files(&self) -> usize { - let mut num = 0; - for file_set in self.file_sets.iter() { - num += file_set.file_paths.len(); - } - num - } -} - -impl EndOutput for ValidateRunResult { - fn output(&self) -> anyhow::Result> { - let mut output: Vec = Vec::new(); - - if let Some(bep_results) = &self.bep_result { - if !bep_results.errors.is_empty() { - output.push(Line::from_iter([ - Span::new_unstyled("⚠️ BEP file had parse errors: ")?, - Span::new_unstyled_lossy(format!("{:?}", bep_results.errors)), - ])); - output.push(Line::default()); - } - } - - output.push(Line::from_iter([Span::new_unstyled(format!( - "πŸ“š Validating {} files:", - self.num_files() - ))?])); - for file_set in self.file_sets.iter() { - output.push(Line::from_iter([Span::new_unstyled(format!( - " Files matching: {}:", - file_set.glob - ))?])); - for file_path in file_set.file_paths.iter() { - output.push(Line::from_iter([Span::new_unstyled(format!( - " {}", - file_path - ))?])) - } - output.push(Line::default()); - } - - output.push(Line::from_iter([Span::new_styled( - style(String::from("File parse issues")).attribute(Attribute::Bold), - )?])); - for file_parse_issue in self.file_parse_issues.iter() { - let has_errors = - file_parse_issue.fatal_error.is_some() || !file_parse_issue.errors.is_empty(); - let has_warnings = !file_parse_issue.warnings.is_empty(); - - let title_colour = if has_errors { - Color::Red - } else if has_warnings { - Color::Yellow - } else { - Color::Green - }; - output.push(Line::from_iter([Span::new_styled_lossy( - style(file_parse_issue.file_path.clone()).with(title_colour), - )])); - - let num_errors = if file_parse_issue.fatal_error.is_some() { - file_parse_issue.errors.len() + 1 - } else { - file_parse_issue.errors.len() - }; - output.push(Line::from_iter([Span::new_styled( - style(format!( - "{} errors, {} warnings", - num_errors, - file_parse_issue.warnings.len() - )) - .attribute(Attribute::Italic), - )?])); - - if has_errors { - output.push(Line::from_iter([Span::new_unstyled(" ❌ Errors:")?])); - if let Some(e) = &file_parse_issue.fatal_error { - output.push(Line::from_iter([Span::new_unstyled_lossy(format!( - " {:?}", - e - ))])); - } - for error in file_parse_issue.errors.iter() { - output.push(Line::from_iter([Span::new_unstyled_lossy(format!( - " {}", - error - ))])); - } - } - - if has_warnings { - output.push(Line::from_iter([Span::new_unstyled(" ⚠️ Warnings:")?])); - for warning in file_parse_issue.warnings.iter() { - output.push(Line::from_iter([Span::new_unstyled_lossy(format!( - " {}", - warning.clone() - ))])); - } - } - - output.push(Line::default()); - } - - output.push(Line::from_iter([Span::new_styled( - style(String::from("Test Validation Issues")).attribute(Attribute::Bold), - )?])); - for test_issue in self.test_issues.iter() { - let has_errors = !test_issue.errors.is_empty(); - let has_warnings = !test_issue.warnings.is_empty(); - - let title_colour = if has_errors { - Color::Red - } else if has_warnings { - Color::Yellow - } else { - Color::Green - }; - output.push(Line::from_iter([Span::new_styled_lossy( - style(test_issue.file_path.clone()).with(title_colour), - )])); - - output.push(Line::from_iter([Span::new_styled( - style(format!( - "{} test suites, {} test cases, {} errors, {} warnings", - test_issue.num_test_suites, - test_issue.num_test_cases, - test_issue.errors.len(), - test_issue.warnings.len(), - )) - .attribute(Attribute::Italic), - )?])); - - if has_errors { - output.push(Line::from_iter([Span::new_unstyled(String::from( - " ❌ Errors:", - ))?])); - for error in test_issue.errors.iter() { - output.push(Line::from_iter([Span::new_unstyled_lossy(format!( - " {}", - error.error_message - ))])); - } - } - - if has_warnings { - output.push(Line::from_iter([Span::new_unstyled(" ⚠️ Warnings:")?])); - for warning in test_issue.warnings.iter() { - output.push(Line::from_iter([Span::new_unstyled_lossy(format!( - " {}", - warning.error_message - ))])); - } - } - } - - output.push(Line::default()); - output.push(Line::from_iter([Span::new_styled( - style(String::from("Checking for codeowners file...")).attribute(Attribute::Bold), - )?])); - match &self.codeowners_issues { - None => { - output.push(Line::from_iter([Span::new_unstyled( - " No codeowners file found", - )?])); - } - Some(codeowners_issues) => { - output.push(Line::from_iter([Span::new_styled_lossy( - style(format!( - " Found codeowners path: {:?}", - codeowners_issues.file_path - )) - .attribute(Attribute::Italic), - )])); - for warning in codeowners_issues.warnings.iter() { - output.push(Line::from_iter([Span::new_unstyled_lossy(format!( - " {}", - warning.clone() - ))])); - } - } - } - output.push(Line::default()); - - let mut num_warnings = 0; - let mut num_errors = 0; - - let mut files_with_no_issues: HashSet = HashSet::new(); - let mut files_with_warnings: HashSet = HashSet::new(); - let mut files_with_errors: HashSet = HashSet::new(); - - for file_set in self.file_sets.iter() { - for file_path in file_set.file_paths.iter() { - files_with_no_issues.insert(file_path.clone()); - } - } - - let num_files = files_with_no_issues.len(); - - for parse_issues in self.file_parse_issues.iter() { - if parse_issues.fatal_error.is_some() { - num_errors += 1; - files_with_errors.insert(parse_issues.file_path.clone()); - files_with_no_issues.remove(&parse_issues.file_path); - } - if !parse_issues.errors.is_empty() { - num_errors += parse_issues.errors.len(); - files_with_errors.insert(parse_issues.file_path.clone()); - files_with_no_issues.remove(&parse_issues.file_path); - } - if !parse_issues.warnings.is_empty() { - num_warnings += parse_issues.warnings.len(); - files_with_warnings.insert(parse_issues.file_path.clone()); - files_with_no_issues.remove(&parse_issues.file_path); - } - } - - for test_issues in self.test_issues.iter() { - if !test_issues.errors.is_empty() { - num_errors += test_issues.errors.len(); - files_with_errors.insert(test_issues.file_path.clone()); - files_with_no_issues.remove(&test_issues.file_path); - } - if !test_issues.warnings.is_empty() { - num_warnings += test_issues.warnings.len(); - files_with_warnings.insert(test_issues.file_path.clone()); - files_with_no_issues.remove(&test_issues.file_path); - } - } - - let num_files_with_no_issues = files_with_no_issues.len(); - let num_files_with_warnings = files_with_warnings.len(); - let num_files_with_errors = files_with_errors.len(); - - output.push(Line::from_iter([ - Span::new_styled(style(format!("{}", num_files_with_no_issues)).with(Color::Green))?, - Span::new_unstyled(format!( - " {}, ", - pluralize("valid file", num_files_with_no_issues as isize, false) - ))?, - Span::new_styled(style(format!("{}", num_files_with_warnings)).with(Color::Yellow))?, - Span::new_unstyled(format!( - " {} with warnings, and ", - pluralize("file", num_files_with_warnings as isize, false) - ))?, - Span::new_styled(style(format!("{}", num_files_with_errors)).with(Color::Red))?, - Span::new_unstyled(format!( - " {} with errors, ", - pluralize("file", num_files_with_errors as isize, false) - ))?, - Span::new_unstyled(format!( - "with {} total (a file is double counted if it has both errors and warnings)", - pluralize("file", num_files as isize, true) - ))?, - ])); - output.push(Line::from_iter([ - Span::new_styled(style(format!("{}", num_warnings)).with(Color::Yellow))?, - Span::new_unstyled(format!( - " {}, and ", - pluralize("warning", num_warnings as isize, false) - ))?, - Span::new_styled(style(format!("{}", num_errors as isize)).with(Color::Red))?, - Span::new_unstyled(format!( - " {}", - pluralize("error", num_errors as isize, false) - ))?, - ])); - - Ok(output) - } + pub(crate) bep_result: Option, + pub(crate) file_sets: Vec, + pub(crate) file_parse_issues: Vec, + pub(crate) test_issues: Vec, + pub(crate) codeowners_issues: Option, + pub(crate) notices: Vec, } fn parse_test_report( @@ -451,10 +155,11 @@ fn flatten_glob(glob_text: &str) -> Vec { } pub async fn run_validate(validate_args: ValidateArgs) -> anyhow::Result { + let mut notices = Vec::new(); if validate_args.hide_banner { - tracing::error!( - "The --hide-banner flag is deprecated and does nothing. It will be removed in a future version." - ); + let message = "The --hide-banner flag is deprecated and does nothing. It will be removed in a future version."; + tracing::error!("{message}"); + notices.push(Notice::deprecation(message)); } let ValidateArgs { @@ -509,6 +214,7 @@ pub async fn run_validate(validate_args: ValidateArgs) -> anyhow::Result, codeowners_type: Option, bep_result: Option, + notices: Vec, ) -> anyhow::Result { let current_dir = std::env::current_dir() .ok() @@ -598,6 +305,7 @@ async fn validate( file_parse_issues, test_issues, codeowners_issues, + notices, }) } From a408da5df056749ad627a717a2ca2bc747958ffa Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 28 Oct 2026 16:08:52 +0000 Subject: [PATCH 08/19] refactor(cli): use the summary_output module path in validate_command Co-Authored-By: Claude Opus 5.5 --- cli/src/validate_command.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/validate_command.rs b/cli/src/validate_command.rs index 9b33fb70..66204f6c 100644 --- a/cli/src/validate_command.rs +++ b/cli/src/validate_command.rs @@ -19,7 +19,7 @@ use context::{ use quick_junit::Report; use crate::{ - context::fall_back_to_binary_parse, error_report::InterruptingError, report::Notice, + context::fall_back_to_binary_parse, error_report::InterruptingError, summary_output::Notice, upload_command::CodeownersType, }; From b9eef070207c8ccd09999ff8e5ceedced3d7607e Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Thu, 22 Oct 2026 22:53:01 +0000 Subject: [PATCH 09/19] feat(cli): add --summary-output-file to upload and test --summary-output-file (or TRUNK_SUMMARY_OUTPUT_FILE) writes the upload report as JSON to a file. The terminal output doesn't change. The file carries a schemaVersion and a type: "upload" with the same report the text is rendered from, or "error" with the ErrorSummary when the run fails outright. The flag is in UploadArgs, so `test` gets it too. The JSON goes to a temporary file next to the target, which is then renamed into place. Missing parent directories are created. On success, "Summary written to " follows the summary on stderr. If the write fails, a warning goes there instead and the exit code doesn't change. Refs TRUNK-22403 Co-Authored-By: Claude Opus 5.5 (1M context) --- cli/src/main.rs | 76 +++++++++++--- cli/src/summary_output/json.rs | 53 ++++++++++ cli/src/summary_output/mod.rs | 1 + cli/src/summary_output/text.rs | 21 ++++ cli/src/test_command.rs | 5 + cli/src/upload_command.rs | 8 ++ cli/tests/summary/error_report.rs | 48 ++++++++- cli/tests/summary/upload.rs | 158 +++++++++++++++++++++++++++++- cli/tests/test.rs | 33 +++++++ cli/tests/upload.rs | 104 ++++++++++++++++++++ constants/src/lib.rs | 2 + 11 files changed, 494 insertions(+), 15 deletions(-) create mode 100644 cli/src/summary_output/json.rs diff --git a/cli/src/main.rs b/cli/src/main.rs index 8462faf9..e77e8c19 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -2,6 +2,7 @@ mod shutdown; use std::{ env, + path::{Path, PathBuf}, sync::{Arc, mpsc::Sender}, thread::JoinHandle, }; @@ -19,7 +20,8 @@ use trunk_analytics_cli::{ error_report::ErrorReport, summary_output::{ ErrorSummary, UploadCommand, UploadReport, ValidateReport, - text::{TextOptions, UploadText}, + json::{self, Report}, + text::{SummaryFileText, TextOptions, UploadText}, }, test_command::{TestArgs, run_test}, upload_command::{RunUploadOptions, UploadArgs, UploadRunResult, run_upload}, @@ -73,6 +75,14 @@ impl Cli { } } + pub fn summary_output_file(&self) -> Option { + match &self.command { + Commands::Test(args) => args.summary_output_file(), + Commands::Upload(args) => args.summary_output_file.clone(), + Commands::Validate(..) => None, + } + } + pub fn repo_root(&self) -> String { let explicit_root = match &self.command { Commands::Test(args) => args.repo_root(), @@ -122,6 +132,7 @@ fn main() -> anyhow::Result<()> { .block_on(async { let cli = Cli::parse(); let org_url_slug = cli.org_url_slug(); + let summary_output_file = cli.summary_output_file(); let log_level_filter = cli.verbose.log_level_filter(); setup_logger( log_level_filter, @@ -152,28 +163,36 @@ fn main() -> anyhow::Result<()> { close_out_and_exit(exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(RunResult::Upload(run_result))) => { - let text = UploadText { + let text = Arc::new(UploadText { report: UploadReport::new(&run_result, UploadCommand::Upload), options: TextOptions::from(&run_result), - }; - let exit_code = text.report.exit_code; + }); send_message( - DisplayMessage::Final(Arc::new(text), String::from("upload display")), + DisplayMessage::Final(text.clone(), String::from("upload display")), &render_sender, ); - close_out_and_exit(exit_code, guard, render_sender, render_handle) + write_summary_file( + summary_output_file.as_deref(), + Report::Upload(&text.report), + &render_sender, + ); + close_out_and_exit(text.report.exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(RunResult::Test(run_result))) => { - let text = UploadText { + let text = Arc::new(UploadText { report: UploadReport::new(&run_result, UploadCommand::Test), options: TextOptions::from(&run_result), - }; - let exit_code = text.report.exit_code; + }); send_message( - DisplayMessage::Final(Arc::new(text), String::from("test display")), + DisplayMessage::Final(text.clone(), String::from("test display")), &render_sender, ); - close_out_and_exit(exit_code, guard, render_sender, render_handle) + write_summary_file( + summary_output_file.as_deref(), + Report::Upload(&text.report), + &render_sender, + ); + close_out_and_exit(text.report.exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(RunResult::Validate(run_result))) => { let report = ValidateReport::new(&run_result); @@ -185,10 +204,19 @@ fn main() -> anyhow::Result<()> { close_out_and_exit(exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Err(error)) => { - let error = ErrorSummary::new(&ErrorReport::new(error, org_url_slug, None)); + let error = Arc::new(ErrorSummary::new(&ErrorReport::new( + error, + org_url_slug, + None, + ))); let exit_code = error.exit_code.unwrap_or(exitcode::OK); send_message( - DisplayMessage::Final(Arc::new(error), String::from("error display")), + DisplayMessage::Final(error.clone(), String::from("error display")), + &render_sender, + ); + write_summary_file( + summary_output_file.as_deref(), + Report::Error(&error), &render_sender, ); close_out_and_exit(exit_code, guard, render_sender, render_handle) @@ -197,6 +225,28 @@ fn main() -> anyhow::Result<()> { }) } +fn write_summary_file(path: Option<&Path>, report: Report, render_sender: &Sender) { + let Some(path) = path else { + return; + }; + let error = json::write_file(path, report).err().map(|error| { + tracing::warn!( + "Failed to write the summary to {}: {:?}", + path.display(), + error + ); + format!("{error:#}") + }); + let text = SummaryFileText { + path: path.to_path_buf(), + error, + }; + send_message( + DisplayMessage::Final(Arc::new(text), String::from("summary file display")), + render_sender, + ); +} + /// Outcome of racing the command against termination signals in `main`. enum MainOutcome { /// The command ran to completion (successfully or with an error). diff --git a/cli/src/summary_output/json.rs b/cli/src/summary_output/json.rs new file mode 100644 index 00000000..dd7867e4 --- /dev/null +++ b/cli/src/summary_output/json.rs @@ -0,0 +1,53 @@ +use std::{fs, path::Path}; + +use serde::Serialize; + +use super::{ErrorSummary, UploadReport}; + +/// Bump when a change could break a reader, such as renaming or removing a field. +pub const SCHEMA_VERSION: u32 = 1; + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum Report<'a> { + Upload(&'a UploadReport), + Error(&'a ErrorSummary), +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Versioned<'a> { + schema_version: u32, + #[serde(flatten)] + report: Report<'a>, +} + +pub fn to_string(report: Report) -> serde_json::Result { + serde_json::to_string_pretty(&Versioned { + schema_version: SCHEMA_VERSION, + report, + }) +} + +/// Writes a sibling file and renames it into place, so a reader never sees half a summary. +/// Not `tempfile::NamedTempFile`, which would leave the summary readable only by its owner. +pub fn write_file(path: &Path, report: Report) -> anyhow::Result<()> { + let json = to_string(report)?; + let file_name = path + .file_name() + .ok_or_else(|| anyhow::anyhow!("{} is not a file path", path.display()))?; + let dir = path.parent().unwrap_or(Path::new("")); + if !dir.as_os_str().is_empty() { + fs::create_dir_all(dir)?; + } + let temp_path = dir.join(format!( + ".{}.{}.tmp", + file_name.to_string_lossy(), + std::process::id() + )); + let written = fs::write(&temp_path, json).and_then(|()| fs::rename(&temp_path, path)); + if written.is_err() { + let _ = fs::remove_file(&temp_path); + } + Ok(written?) +} diff --git a/cli/src/summary_output/mod.rs b/cli/src/summary_output/mod.rs index 4de156fa..0a4c899e 100644 --- a/cli/src/summary_output/mod.rs +++ b/cli/src/summary_output/mod.rs @@ -1,4 +1,5 @@ mod error; +pub mod json; mod notice; pub mod text; mod upload; diff --git a/cli/src/summary_output/text.rs b/cli/src/summary_output/text.rs index 845ebc5d..9d45dc0a 100644 --- a/cli/src/summary_output/text.rs +++ b/cli/src/summary_output/text.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use constants::EXIT_SUCCESS; use display::end_output::EndOutput; use pluralizer::pluralize; @@ -53,6 +55,25 @@ impl EndOutput for ErrorSummary { } } +pub struct SummaryFileText { + pub path: PathBuf, + pub error: Option, +} + +impl EndOutput for SummaryFileText { + fn output(&self) -> anyhow::Result> { + let path = self.path.display(); + let message = match &self.error { + None => format!("Summary written to {path}"), + Some(error) => format!("⚠️ Failed to write the summary to {path}: {error}"), + }; + Ok(vec![ + Line::default(), + Line::from_iter([Span::new_unstyled_lossy(message)]), + ]) + } +} + pub fn render_error(error: &ErrorSummary) -> anyhow::Result> { let mut lines = Vec::new(); lines.push(Line::from_iter([Span::new_styled( diff --git a/cli/src/test_command.rs b/cli/src/test_command.rs index 868942cb..98c2c071 100644 --- a/cli/src/test_command.rs +++ b/cli/src/test_command.rs @@ -1,5 +1,6 @@ use std::{ env, + path::PathBuf, process::{Command, Stdio}, sync::{Arc, mpsc::Sender}, time::SystemTime, @@ -62,6 +63,10 @@ impl TestArgs { pub fn repo_root(&self) -> Option { self.upload_args.repo_root.clone() } + + pub fn summary_output_file(&self) -> Option { + self.upload_args.summary_output_file.clone() + } } #[derive(Debug, Clone)] diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index 48861e7a..a07ab776 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -393,6 +393,14 @@ pub struct UploadArgs { default_missing_value = "true" )] pub hide_test_collection_links: bool, + #[arg( + long, + env = constants::TRUNK_SUMMARY_OUTPUT_FILE_ENV, + help = "Also write the summary as JSON to this file, e.g. for a step that comments on the pull request.", + required = false, + num_args = 1 + )] + pub summary_output_file: Option, } #[derive(clap::ValueEnum, Clone, Copy, Debug, Default, PartialEq, Eq)] diff --git a/cli/tests/summary/error_report.rs b/cli/tests/summary/error_report.rs index f916e2fb..5d7c1c07 100644 --- a/cli/tests/summary/error_report.rs +++ b/cli/tests/summary/error_report.rs @@ -3,7 +3,11 @@ use pretty_assertions::assert_eq; use superconsole::Line; use trunk_analytics_cli::{ error_report::{ErrorReport, InterruptingError}, - summary_output::{ErrorSummary, text}, + summary_output::{ + ErrorSummary, + json::{self, Report}, + text, + }, }; use crate::unauthorized_error; @@ -182,3 +186,45 @@ fn prints_any_other_exit_code() { ] ); } + +#[test] +fn writes_an_error_as_json() { + let report = ErrorReport::new( + anyhow::anyhow!("--allow-forked-pr-uploads requires --test-collection-id") + .context("Failed to gather upload context"), + String::from("fake-org-slug"), + None, + ); + let error = ErrorSummary::new(&report); + + assert_eq!( + serde_json::from_str::(&json::to_string(Report::Error(&error)).unwrap()) + .unwrap(), + serde_json::json!({ + "schemaVersion": 1, + "type": "error", + "kind": "other", + "message": null, + "error": "Failed to gather upload context", + "causes": ["--allow-forked-pr-uploads requires --test-collection-id"], + "explanation": null, + "settingsUrl": null, + "exitCode": null, + }) + ); +} + +#[test] +fn writes_the_settings_url_and_exit_code_of_an_unauthorized_error_as_json() { + let report = ErrorReport::new(unauthorized_error(), String::from("fake-org-slug"), None); + let error = ErrorSummary::new(&report); + let json: serde_json::Value = + serde_json::from_str(&json::to_string(Report::Error(&error)).unwrap()).unwrap(); + + assert_eq!(json["kind"], "unauthorized"); + assert_eq!( + json["settingsUrl"], + "https://app.trunk.io/fake-org-slug/settings" + ); + assert_eq!(json["exitCode"], 70); +} diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index dae9d82b..6935cf71 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -20,7 +20,8 @@ use trunk_analytics_cli::{ error_report::ErrorReport, report_limiting::ValidationReport, summary_output::{ - UploadCommand, UploadReport, + Notice, UploadCommand, UploadReport, + json::{self, Report}, text::{self, TextOptions}, }, upload_command::UploadRunResult, @@ -855,3 +856,158 @@ fn exits_with_the_test_run_exit_code_when_an_error_has_none() { ); assert_eq!(UploadReport::new(&result, UploadCommand::Test).exit_code, 1); } + +fn upload_json(result: &UploadRunResult) -> serde_json::Value { + let report = UploadReport::new(result, UploadCommand::Upload); + serde_json::from_str(&json::to_string(Report::Upload(&report)).unwrap()).unwrap() +} + +#[test] +fn writes_the_report_as_json() { + let mut result = in_collection(upload_result(10)); + result.error_report = Some(ErrorReport::new( + anyhow::anyhow!("HTTP status server error (500 Internal Server Error)") + .context("Failed to create bundle upload"), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + result.validations = JunitReportValidations::new(BTreeMap::from([ + ( + String::from("test-results/a.xml"), + validation(vec![ + name_too_short(), + warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration), + ]), + ), + ( + String::from("test-results/b.xml"), + Err(anyhow::anyhow!("unexpected end of file")), + ), + ])); + result.quarantine_context.exit_code = 1; + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![failure("adds", "MathTests", Some("src/math.rs"))]; + result.quarantine_context.failures = vec![ + Test { + class_name: Some(String::from("Math")), + failure_message: Some(String::from("expected 1, got 2")), + variant: Some(String::from("linux")), + ..failure("subtracts", "MathTests", Some("src/math.rs")) + }, + failure("boots", "", None), + ]; + result.notices = vec![Notice::deprecation( + "The --team flag is deprecated and will be removed in a future version.", + )]; + + assert_eq!( + upload_json(&result), + serde_json::json!({ + "schemaVersion": 1, + "type": "upload", + "error": { + "kind": "other", + "message": "There was an unexpected error that occurred while uploading test results", + "error": "Failed to create bundle upload", + "causes": ["HTTP status server error (500 Internal Server Error)"], + "explanation": null, + "settingsUrl": null, + "exitCode": null, + }, + "bundleUploadId": "4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "uploadUrl": "https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", + "validations": [ + { + "file": "test-results/a.xml", + "parseError": null, + "errors": ["test case name too short"], + "warnings": ["test case or parent has no time duration"], + "testSuiteCount": 0, + "testCaseCount": 0, + }, + { + "file": "test-results/b.xml", + "parseError": { + "error": "unexpected end of file", + "causes": [], + }, + "errors": [], + "warnings": [], + "testSuiteCount": null, + "testCaseCount": null, + }, + ], + "counts": { + "total": 10, + "passed": 7, + "failed": 2, + "quarantined": 1, + "passRatio": 0.7, + }, + "quarantineLookupFailed": false, + "quarantined": [ + { + "key": { "file": "src/math.rs" }, + "tests": [ + { + "name": "adds", + "suite": "MathTests", + "className": null, + "file": "src/math.rs", + "id": "adds-id", + "variant": null, + "failureMessage": null, + "quarantined": true, + "link": "https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/adds-id?repo=trunk-io%2Fanalytics-cli", + }, + ], + }, + ], + "failures": [ + { + "key": { "file": "src/math.rs" }, + "tests": [ + { + "name": "subtracts", + "suite": "MathTests", + "className": "Math", + "file": "src/math.rs", + "id": "subtracts-id", + "variant": "linux", + "failureMessage": "expected 1, got 2", + "quarantined": false, + "link": "https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/subtracts-id?repo=trunk-io%2Fanalytics-cli", + }, + ], + }, + { + "key": "other", + "tests": [ + { + "name": "boots", + "suite": "", + "className": null, + "file": null, + "id": "boots-id", + "variant": null, + "failureMessage": null, + "quarantined": false, + "link": "https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/boots-id?repo=trunk-io%2Fanalytics-cli", + }, + ], + }, + ], + "outcome": "failuresNotQuarantined", + "exitCode": 1, + "notices": [ + { + "kind": "deprecation", + "message": "The --team flag is deprecated and will be removed in a future version.", + }, + ], + }) + ); +} diff --git a/cli/tests/test.rs b/cli/tests/test.rs index 5901b164..9c43e508 100644 --- a/cli/tests/test.rs +++ b/cli/tests/test.rs @@ -554,3 +554,36 @@ async fn test_command_can_print_subcommand_output_with_faux_html() { "This breaks superconsole", )); } + +#[tokio::test(flavor = "multi_thread")] +async fn test_command_writes_summary_output_file() { + let temp_dir = tempdir().unwrap(); + generate_mock_git_repo(&temp_dir); + let summary_path = temp_dir.path().join("summary.json"); + + let state = MockServerBuilder::new().spawn_mock_server().await; + + CommandBuilder::test( + temp_dir.path(), + state.host.clone(), + vec![ + String::from("bash"), + String::from("-c"), + String::from("exit 1"), + ], + ) + .command() + .env(constants::TRUNK_SUMMARY_OUTPUT_FILE_ENV, &summary_path) + .assert() + .code(1) + .stderr(predicate::str::contains(format!( + "Summary written to {}", + summary_path.display() + ))); + + let summary: serde_json::Value = + serde_json::from_reader(File::open(&summary_path).unwrap()).unwrap(); + assert_eq!(summary["schemaVersion"], 1); + assert_eq!(summary["type"], "upload"); + assert_eq!(summary["exitCode"], 1); +} diff --git a/cli/tests/upload.rs b/cli/tests/upload.rs index 5a8c1098..6a6bcf61 100644 --- a/cli/tests/upload.rs +++ b/cli/tests/upload.rs @@ -1528,6 +1528,110 @@ async fn upload_bundle_invalid_repo_root_explicit() { println!("{assert}"); } +fn read_summary_output_file(path: &std::path::Path) -> serde_json::Value { + serde_json::from_reader(fs::File::open(path).unwrap()).unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +async fn upload_writes_summary_output_file() { + let temp_dir = tempdir().unwrap(); + generate_mock_git_repo(&temp_dir); + generate_mock_valid_junit_xmls_with_failures(&temp_dir); + let summary_path = temp_dir.path().join("out").join("summary.json"); + + let state = MockServerBuilder::new().spawn_mock_server().await; + let assert = CommandBuilder::upload(temp_dir.path(), state.host.clone()) + .command() + .arg("--summary-output-file") + .arg(&summary_path) + .assert() + .failure() + .stderr(predicate::str::contains(format!( + "Summary written to {}", + summary_path.display() + ))); + + let summary = read_summary_output_file(&summary_path); + assert_eq!(summary["schemaVersion"], 1); + assert_eq!(summary["type"], "upload"); + assert_eq!( + summary["exitCode"], + assert.get_output().status.code().unwrap() + ); + let counts = &summary["counts"]; + let count = |key: &str| counts[key].as_u64().unwrap(); + assert!(count("failed") > 0); + assert_eq!( + count("passed") + count("failed") + count("quarantined"), + count("total") + ); + let failures: Vec<&serde_json::Value> = summary["failures"] + .as_array() + .unwrap() + .iter() + .flat_map(|group| group["tests"].as_array().unwrap()) + .collect(); + assert_eq!(failures.len() as u64, count("failed")); + for failure in failures { + assert!(!failure["name"].as_str().unwrap().is_empty()); + assert!( + failure["link"] + .as_str() + .unwrap() + .contains("/flaky-tests/test/") + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn upload_writes_an_error_to_summary_output_file() { + let temp_dir = tempdir().unwrap(); + generate_mock_git_repo(&temp_dir); + let summary_path = temp_dir.path().join("summary.json"); + + let state = MockServerBuilder::new().spawn_mock_server().await; + CommandBuilder::upload(temp_dir.path(), state.host.clone()) + .repo_root("../") + .command() + .arg("--summary-output-file") + .arg(&summary_path) + .assert() + .code(exitcode::SOFTWARE); + + let summary = read_summary_output_file(&summary_path); + assert_eq!(summary["schemaVersion"], 1); + assert_eq!(summary["type"], "error"); + assert_eq!(summary["kind"], "gitRepository"); + assert_eq!(summary["exitCode"], exitcode::SOFTWARE); +} + +#[tokio::test(flavor = "multi_thread")] +async fn upload_keeps_its_exit_code_when_the_summary_output_file_cannot_be_written() { + let temp_dir = tempdir().unwrap(); + generate_mock_git_repo(&temp_dir); + let summary_path = temp_dir.path().join("summary.json"); + fs::create_dir(&summary_path).unwrap(); + + let state = MockServerBuilder::new().spawn_mock_server().await; + CommandBuilder::upload(temp_dir.path(), state.host.clone()) + .command() + .arg("--summary-output-file") + .arg(&summary_path) + .assert() + .success() + .stderr(predicate::str::contains(format!( + "Failed to write the summary to {}", + summary_path.display() + ))); + + let leftovers: Vec<_> = fs::read_dir(temp_dir.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| name.to_string_lossy().ends_with(".tmp")) + .collect(); + assert!(leftovers.is_empty(), "left behind {leftovers:?}"); +} + #[tokio::test(flavor = "multi_thread")] async fn upload_bundle_valid_repo_root_implicit() { let temp_dir = tempdir().unwrap(); diff --git a/constants/src/lib.rs b/constants/src/lib.rs index 6024ad69..b726fc70 100644 --- a/constants/src/lib.rs +++ b/constants/src/lib.rs @@ -58,6 +58,7 @@ pub const TRUNK_TEST_PROCESS_EXIT_CODE_ENV: &str = "TRUNK_TEST_PROCESS_EXIT_CODE pub const TRUNK_VALIDATION_REPORT_ENV: &str = "TRUNK_VALIDATION_REPORT"; pub const TRUNK_SHOW_FAILURE_MESSAGES_ENV: &str = "TRUNK_SHOW_FAILURE_MESSAGES"; pub const TRUNK_HIDE_TEST_COLLECTION_LINKS_ENV: &str = "TRUNK_HIDE_TEST_COLLECTION_LINKS"; +pub const TRUNK_SUMMARY_OUTPUT_FILE_ENV: &str = "TRUNK_SUMMARY_OUTPUT_FILE"; pub const TRUNK_DEBUG_ENV: &str = "TRUNK_DEBUG"; pub const TRUNK_USE_EXPERIMENTAL_XCRESULT_TEST_LOCATIONS_ENV: &str = "TRUNK_USE_EXPERIMENTAL_XCRESULT_TEST_LOCATIONS"; @@ -111,6 +112,7 @@ pub const TRUNK_ENVS_TO_CAPTURE: &[&str] = &[ TRUNK_VALIDATION_REPORT_ENV, TRUNK_SHOW_FAILURE_MESSAGES_ENV, TRUNK_HIDE_TEST_COLLECTION_LINKS_ENV, + TRUNK_SUMMARY_OUTPUT_FILE_ENV, TRUNK_DEBUG_ENV, TRUNK_USE_EXPERIMENTAL_XCRESULT_TEST_LOCATIONS_ENV, ]; From 7ee6db7c093d2fd674d488ad0d768817e3539c35 Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Thu, 22 Oct 2026 23:13:46 +0000 Subject: [PATCH 10/19] feat(cli): add summary-output-schema to print the summary's JSON Schema `trunk-analytics-cli summary-output-schema` prints the JSON Schema for what --summary-output-file writes. schemars generates it from the report types, so it follows the serde attributes (camelCase, the `type` tag) and takes its descriptions from their doc comments. It is generated for serialization, so the fields that are always written, even as null, are required. Refs TRUNK-22403 Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 69 +++++++++++++++++++++++++++++--- cli/Cargo.toml | 1 + cli/src/error_report.rs | 7 +++- cli/src/main.rs | 17 ++++++-- cli/src/summary_output/error.rs | 9 ++++- cli/src/summary_output/json.rs | 20 +++++++-- cli/src/summary_output/notice.rs | 7 +++- cli/src/summary_output/upload.rs | 47 +++++++++++++++++----- cli/tests/summary/main.rs | 1 + cli/tests/summary/schema.rs | 19 +++++++++ 10 files changed, 171 insertions(+), 26 deletions(-) create mode 100644 cli/tests/summary/schema.rs diff --git a/Cargo.lock b/Cargo.lock index b391e8a1..68015815 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4413,6 +4413,26 @@ dependencies = [ "bitflags 2.10.0", ] +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "regex" version = "1.12.2" @@ -4679,7 +4699,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ "dyn-clone", - "schemars_derive", + "schemars_derive 0.8.22", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive 1.2.3", "serde", "serde_json", ] @@ -4692,10 +4725,22 @@ checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" dependencies = [ "proc-macro2", "quote", - "serde_derive_internals", + "serde_derive_internals 0.29.1", "syn 2.0.110", ] +[[package]] +name = "schemars_derive" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals 0.30.0", + "syn 3.0.4", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4909,6 +4954,17 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "serde_json" version = "1.0.145" @@ -5919,6 +5975,7 @@ dependencies = [ "regex", "reqwest", "rstest", + "schemars 1.2.3", "sentry", "sentry-tracing", "serde", @@ -5961,7 +6018,7 @@ checksum = "f81253930d0d388a3ab8fa4ae56da9973ab171ef833d1be2e9080fc3ce502bd6" dependencies = [ "proc-macro2", "quote", - "serde_derive_internals", + "serde_derive_internals 0.29.1", "syn 2.0.110", ] @@ -6022,7 +6079,7 @@ dependencies = [ "proc-macro2", "quote", "regress", - "schemars", + "schemars 0.8.22", "semver", "serde", "serde_json", @@ -6039,7 +6096,7 @@ checksum = "68b5780d745920ed73c5b7447496a9b5c42ed2681a9b70859377aec423ecf02b" dependencies = [ "proc-macro2", "quote", - "schemars", + "schemars 0.8.22", "semver", "serde", "serde_json", @@ -6678,7 +6735,7 @@ dependencies = [ "quick-junit", "regex", "rstest", - "schemars", + "schemars 0.8.22", "serde", "serde_json", "syn 2.0.110", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 5455215c..8667c4c2 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -50,6 +50,7 @@ sentry-tracing = "0.36.0" openssl = { version = "0.10.78", features = ["vendored"] } openssl-src = { version = "=300.3.1", optional = true } quick-junit = "0.5.0" +schemars = "1.2" serde = { version = "1.0.215", features = ["derive"] } serde_json = "1.0" clap-verbosity-flag = "3.0.2" diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index 405b6702..4c668203 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -1,5 +1,6 @@ use api::client::ClientErrorExplanation; use http::StatusCode; +use schemars::JsonSchema; use serde::Serialize; pub(crate) const HELP_TEXT: &str = "For more help, contact us at https://slack.trunk.io/"; @@ -77,12 +78,16 @@ impl ErrorReport { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum ErrorKind { + /// Couldn't connect to Trunk. ConnectionRefused, + /// Trunk rejected the org slug or token. Unauthorized, + /// The repo root couldn't be opened as a git repository. GitRepository, + /// The run stopped early, such as when it found no test reports and empty results aren't allowed. Interrupted, Other, } diff --git a/cli/src/main.rs b/cli/src/main.rs index e77e8c19..518dbaee 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -45,7 +45,7 @@ impl Cli { let token = match &self.command { Commands::Test(args) => args.token(), Commands::Upload(args) => args.token.clone(), - Commands::Validate(..) => None, + Commands::Validate(..) | Commands::SummaryOutputSchema => None, }; token.map_or( @@ -59,6 +59,7 @@ impl Cli { Commands::Test(..) => "test", Commands::Upload(..) => "upload", Commands::Validate(..) => "validate", + Commands::SummaryOutputSchema => "summary-output-schema", } } @@ -71,7 +72,7 @@ impl Cli { match &self.command { Commands::Test(args) => args.org_url_slug(), Commands::Upload(args) => args.org_url_slug.clone(), - Commands::Validate(..) => String::from("not used"), + Commands::Validate(..) | Commands::SummaryOutputSchema => String::from("not used"), } } @@ -79,7 +80,7 @@ impl Cli { match &self.command { Commands::Test(args) => args.summary_output_file(), Commands::Upload(args) => args.summary_output_file.clone(), - Commands::Validate(..) => None, + Commands::Validate(..) | Commands::SummaryOutputSchema => None, } } @@ -87,7 +88,7 @@ impl Cli { let explicit_root = match &self.command { Commands::Test(args) => args.repo_root(), Commands::Upload(args) => args.repo_root.clone(), - Commands::Validate(..) => None, + Commands::Validate(..) | Commands::SummaryOutputSchema => None, }; explicit_root .or(std::env::current_dir() @@ -106,6 +107,8 @@ enum Commands { Upload(UploadArgs), /// Validate test report files for compatibility with Trunk Flaky Tests. Runs locally without uploading data. Validate(ValidateArgs), + /// Print the JSON Schema of the summary that `--summary-output-file` writes. + SummaryOutputSchema, } fn close_out_and_exit( @@ -203,6 +206,10 @@ fn main() -> anyhow::Result<()> { ); close_out_and_exit(exit_code, guard, render_sender, render_handle) } + MainOutcome::Finished(Ok(RunResult::SummaryOutputSchema)) => { + println!("{}", serde_json::to_string_pretty(&json::schema())?); + close_out_and_exit(exitcode::OK, guard, render_sender, render_handle) + } MainOutcome::Finished(Err(error)) => { let error = Arc::new(ErrorSummary::new(&ErrorReport::new( error, @@ -259,6 +266,7 @@ enum RunResult { Upload(UploadRunResult), Test(UploadRunResult), Validate(ValidateRunResult), + SummaryOutputSchema, } async fn run(cli: Cli, render_sender: Sender) -> anyhow::Result { @@ -288,6 +296,7 @@ async fn run(cli: Cli, render_sender: Sender) -> anyhow::Result< let result = run_validate(validate_args).await?; Ok(RunResult::Validate(result)) } + Commands::SummaryOutputSchema => Ok(RunResult::SummaryOutputSchema), } } diff --git a/cli/src/summary_output/error.rs b/cli/src/summary_output/error.rs index 316d2b16..c0a3d91a 100644 --- a/cli/src/summary_output/error.rs +++ b/cli/src/summary_output/error.rs @@ -1,17 +1,24 @@ use api::client::get_api_host; +use schemars::JsonSchema; use serde::Serialize; use crate::error_report::{ErrorKind, ErrorReport, get_client_error_explanation, settings_url}; -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ErrorSummary { pub kind: ErrorKind, + /// What the CLI was doing when the error happened. pub message: Option, + /// The error, without its causes. pub error: String, + /// The errors that led to it, outermost first. pub causes: Vec, + /// Trunk's explanation, when its API gave one. pub explanation: Option, + /// Where to check the org slug and token. Only set for `unauthorized` errors. pub settings_url: Option, + /// A run that fails outright exits with this code, or 0 when it's null. Upload reports exit with their own `exitCode`. pub exit_code: Option, } diff --git a/cli/src/summary_output/json.rs b/cli/src/summary_output/json.rs index dd7867e4..e52a0e63 100644 --- a/cli/src/summary_output/json.rs +++ b/cli/src/summary_output/json.rs @@ -1,27 +1,41 @@ use std::{fs, path::Path}; +use schemars::{JsonSchema, Schema, generate::SchemaSettings}; use serde::Serialize; use super::{ErrorSummary, UploadReport}; -/// Bump when a change could break a reader, such as renaming or removing a field. pub const SCHEMA_VERSION: u32 = 1; -#[derive(Debug, Clone, Copy, Serialize)] +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] #[serde(tag = "type", rename_all = "camelCase")] pub enum Report<'a> { + /// The run finished and summarized its tests. Upload(&'a UploadReport), + /// The run failed before it could summarize any tests. Error(&'a ErrorSummary), } -#[derive(Serialize)] +#[derive(Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] +#[schemars( + title = "Trunk Analytics CLI summary", + description = "What `upload` and `test` write to `--summary-output-file`." +)] struct Versioned<'a> { + /// Goes up when a change could break a reader, such as renaming or removing a field. schema_version: u32, #[serde(flatten)] report: Report<'a>, } +pub fn schema() -> Schema { + SchemaSettings::draft2020_12() + .for_serialize() + .into_generator() + .into_root_schema_for::>() +} + pub fn to_string(report: Report) -> serde_json::Result { serde_json::to_string_pretty(&Versioned { schema_version: SCHEMA_VERSION, diff --git a/cli/src/summary_output/notice.rs b/cli/src/summary_output/notice.rs index 763c6bec..c5a02848 100644 --- a/cli/src/summary_output/notice.rs +++ b/cli/src/summary_output/notice.rs @@ -1,16 +1,19 @@ +use schemars::JsonSchema; use serde::Serialize; -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct Notice { pub kind: NoticeKind, pub message: String, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum NoticeKind { + /// A deprecated flag was passed. Deprecation, + /// Part of the Bazel BEP file couldn't be parsed. BepParseError, } diff --git a/cli/src/summary_output/upload.rs b/cli/src/summary_output/upload.rs index 3388af5d..667daa8e 100644 --- a/cli/src/summary_output/upload.rs +++ b/cli/src/summary_output/upload.rs @@ -4,46 +4,60 @@ use api::urls::{url_for_test_case, url_for_upload}; use bundle::Test; use constants::EXIT_SUCCESS; use context::junit::validator::JunitValidationLevel; +use schemars::JsonSchema; use serde::Serialize; use super::{ErrorSummary, Notice}; use crate::{upload_command::UploadRunResult, validate_command::JunitReportValidations}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum UploadCommand { Upload, Test, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct UploadReport { + /// Which command wrote this summary. pub command: UploadCommand, + /// Why uploading the results failed. The tests are still summarized. pub error: Option, pub bundle_upload_id: Option, + /// This upload's page in Trunk. Only set for uploads to a test collection. pub upload_url: Option, + /// One entry per test report file. pub validations: Vec, pub counts: TestCounts, + /// Trunk couldn't say which tests are quarantined, so no failed test counts as quarantined. pub quarantine_lookup_failed: bool, + /// Failed tests that were quarantined, grouped like `failures`. pub quarantined: Vec, + /// Failed tests that weren't quarantined, grouped by file, else by suite. pub failures: Vec, pub outcome: Outcome, + /// The code the CLI exits with. pub exit_code: i32, + /// Warnings that didn't stop the run, such as a deprecated flag. pub notices: Vec, } -#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct TestCounts { pub total: usize, + /// Tests that neither failed nor were quarantined. pub passed: usize, + /// Failed tests that weren't quarantined. pub failed: usize, + /// Failed tests that were quarantined. pub quarantined: usize, + /// `passed / total`, from 0 to 1. Null when there are no tests. pub pass_ratio: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct FailureGroup { pub key: GroupKey, @@ -51,21 +65,26 @@ pub struct FailureGroup { } /// Groups sort by file, then by suite, then the rest, in declaration order. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum GroupKey { + /// Failed tests from this file. File(String), + /// Failed tests from this suite that have no file. Suite(String), + /// Failed tests with neither a file nor a suite. Other, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct FailedTest { pub name: String, + /// Empty when the test has no suite. pub suite: String, pub class_name: Option, pub file: Option, + /// Trunk's ID for the test. pub id: String, pub variant: Option, /// The failure's `message` attribute. @@ -73,34 +92,44 @@ pub struct FailedTest { /// The failure's text, usually a stack trace. pub failure_text: Option, pub quarantined: bool, + /// The test's page in Trunk. pub link: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct FileValidation { pub file: String, + /// Why the file couldn't be parsed. `errors` and `warnings` are empty when this is set. pub parse_error: Option, pub errors: Vec, pub warnings: Vec, + /// Null when the file couldn't be parsed. pub test_suite_count: Option, + /// Null when the file couldn't be parsed. pub test_case_count: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ParseError { pub error: String, + /// The errors that caused `error`, outermost first. pub causes: Vec, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum Outcome { + /// The test reports had no tests. NoTestsFound, + /// Every failed test was quarantined, so none of them fail the run. AllFailuresQuarantined, + /// No tests failed. NoFailures, + /// No tests failed, but the test run's exit code wasn't 0, so some tests may not have run. NoFailuresButNonZeroExitCode, + /// Some failed tests weren't quarantined. FailuresNotQuarantined, } diff --git a/cli/tests/summary/main.rs b/cli/tests/summary/main.rs index ad3c87ed..07ecdf68 100644 --- a/cli/tests/summary/main.rs +++ b/cli/tests/summary/main.rs @@ -1,4 +1,5 @@ mod error_report; +mod schema; mod upload; mod validate; diff --git a/cli/tests/summary/schema.rs b/cli/tests/summary/schema.rs new file mode 100644 index 00000000..ba522d7a --- /dev/null +++ b/cli/tests/summary/schema.rs @@ -0,0 +1,19 @@ +use assert_cmd::{Command, cargo}; +use pretty_assertions::assert_eq; +use trunk_analytics_cli::report::json; + +#[test] +fn prints_the_summary_output_schema() { + let output = Command::new(cargo::cargo_bin!()) + .arg("summary-output-schema") + .assert() + .success() + .get_output() + .stdout + .clone(); + + assert_eq!( + serde_json::from_slice::(&output).unwrap(), + serde_json::to_value(json::schema()).unwrap() + ); +} From 256b64028c1516bc296b0c4f391646103a827ef8 Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Fri, 23 Oct 2026 18:36:28 +0000 Subject: [PATCH 11/19] feat(rspec): write the summary to TRUNK_SUMMARY_OUTPUT_FILE When TRUNK_SUMMARY_OUTPUT_FILE is set, the RSpec gem writes the same JSON summary as `upload --summary-output-file` after uploading, or the error object when the upload fails outright. "Summary written to " goes to stderr, as with the CLI, because the gem's own logs only reach the console at error level. The gem's summary has no exitCode, because RSpec decides the exit code. UploadCommand::RSpec leaves the report's exit_code unset, and the field is skipped when unset. `upload` and `test` always set it. The upload trusts the gem's quarantine results without looking them up again, so quarantineLookupFailed comes from the gem's own lookup. ErrorReport::unlogged builds the error object without logging it, since the gem already logs that error to Sentry. Refs TRUNK-22403 Co-Authored-By: Claude Opus 5.5 (1M context) --- cli/src/error_report.rs | 24 ++- cli/src/main.rs | 27 ++-- cli/src/summary_output/json.rs | 2 +- cli/src/summary_output/text.rs | 42 +++-- cli/src/summary_output/upload.rs | 29 +++- cli/tests/summary/schema.rs | 2 +- cli/tests/summary/upload.rs | 14 +- rspec-trunk-flaky-tests/README.md | 8 + .../lib/trunk_spec_helper.rb | 1 + test_report/src/report.rs | 40 ++++- test_report/tests/common/env.rs | 6 +- test_report/tests/publish.rs | 148 +++++++++++++++++- 12 files changed, 295 insertions(+), 48 deletions(-) diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index 4c668203..140ad170 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -51,21 +51,35 @@ pub struct ErrorReport { impl ErrorReport { pub fn new(error: anyhow::Error, org_url_slug: String, base_message: Option) -> Self { - let kind = ErrorKind::of(&error); - match kind { + let report = Self::unlogged(error, org_url_slug, base_message); + match report.kind { ErrorKind::ConnectionRefused => tracing::warn!(CONNECTION_REFUSED_CONTEXT), ErrorKind::Unauthorized => tracing::warn!(UNAUTHORIZED_CONTEXT), ErrorKind::GitRepository => tracing::warn!(GIX_ERROR_CONTEXT), ErrorKind::Interrupted => { - if let Some(message) = get_interrupting_message(&error) { + if let Some(message) = get_interrupting_message(&report.error) { tracing::warn!("{}", message); } } ErrorKind::Other => { - tracing::error!("{}", error); - tracing::error!(hidden_in_console = true, "Caused by error: {:#?}", error); + tracing::error!("{}", report.error); + tracing::error!( + hidden_in_console = true, + "Caused by error: {:#?}", + report.error + ); } } + report + } + + /// For callers that log the error themselves, so it isn't sent to Sentry twice. + pub fn unlogged( + error: anyhow::Error, + org_url_slug: String, + base_message: Option, + ) -> Self { + let kind = ErrorKind::of(&error); Self { context: Context { base_message, diff --git a/cli/src/main.rs b/cli/src/main.rs index 518dbaee..2817e5d6 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -179,7 +179,12 @@ fn main() -> anyhow::Result<()> { Report::Upload(&text.report), &render_sender, ); - close_out_and_exit(text.report.exit_code, guard, render_sender, render_handle) + close_out_and_exit( + text.report.cli_exit_code(), + guard, + render_sender, + render_handle, + ) } MainOutcome::Finished(Ok(RunResult::Test(run_result))) => { let text = Arc::new(UploadText { @@ -195,7 +200,12 @@ fn main() -> anyhow::Result<()> { Report::Upload(&text.report), &render_sender, ); - close_out_and_exit(text.report.exit_code, guard, render_sender, render_handle) + close_out_and_exit( + text.report.cli_exit_code(), + guard, + render_sender, + render_handle, + ) } MainOutcome::Finished(Ok(RunResult::Validate(run_result))) => { let report = ValidateReport::new(&run_result); @@ -236,18 +246,7 @@ fn write_summary_file(path: Option<&Path>, report: Report, render_sender: &Sende let Some(path) = path else { return; }; - let error = json::write_file(path, report).err().map(|error| { - tracing::warn!( - "Failed to write the summary to {}: {:?}", - path.display(), - error - ); - format!("{error:#}") - }); - let text = SummaryFileText { - path: path.to_path_buf(), - error, - }; + let text = SummaryFileText::new(path, json::write_file(path, report)); send_message( DisplayMessage::Final(Arc::new(text), String::from("summary file display")), render_sender, diff --git a/cli/src/summary_output/json.rs b/cli/src/summary_output/json.rs index e52a0e63..c5da3008 100644 --- a/cli/src/summary_output/json.rs +++ b/cli/src/summary_output/json.rs @@ -20,7 +20,7 @@ pub enum Report<'a> { #[serde(rename_all = "camelCase")] #[schemars( title = "Trunk Analytics CLI summary", - description = "What `upload` and `test` write to `--summary-output-file`." + description = "What `upload` and `test` write to `--summary-output-file`, and the RSpec gem to `TRUNK_SUMMARY_OUTPUT_FILE`." )] struct Versioned<'a> { /// Goes up when a change could break a reader, such as renaming or removing a field. diff --git a/cli/src/summary_output/text.rs b/cli/src/summary_output/text.rs index 9d45dc0a..e1f41ea8 100644 --- a/cli/src/summary_output/text.rs +++ b/cli/src/summary_output/text.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use constants::EXIT_SUCCESS; use display::end_output::EndOutput; @@ -56,20 +56,40 @@ impl EndOutput for ErrorSummary { } pub struct SummaryFileText { - pub path: PathBuf, - pub error: Option, + path: PathBuf, + error: Option, } -impl EndOutput for SummaryFileText { - fn output(&self) -> anyhow::Result> { +impl SummaryFileText { + pub fn new(path: &Path, written: anyhow::Result<()>) -> Self { + let error = written.err().map(|error| { + tracing::warn!( + "Failed to write the summary to {}: {:?}", + path.display(), + error + ); + format!("{error:#}") + }); + Self { + path: path.to_path_buf(), + error, + } + } + + pub fn message(&self) -> String { let path = self.path.display(); - let message = match &self.error { + match &self.error { None => format!("Summary written to {path}"), Some(error) => format!("⚠️ Failed to write the summary to {path}: {error}"), - }; + } + } +} + +impl EndOutput for SummaryFileText { + fn output(&self) -> anyhow::Result> { Ok(vec![ Line::default(), - Line::from_iter([Span::new_unstyled_lossy(message)]), + Line::from_iter([Span::new_unstyled_lossy(self.message())]), ]) } } @@ -268,7 +288,7 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re "⚠️ No test failures found, but non zero exit code provided: ", )?, Span::new_styled( - style(format!("{}", report.exit_code)).attribute(Attribute::Bold), + style(format!("{}", report.cli_exit_code())).attribute(Attribute::Bold), )?, ])); output.push(Line::from_iter([Span::new_unstyled( @@ -279,7 +299,9 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re Span::new_unstyled( "⚠️ Some test failures were not quarantined, using exit code: ".to_string(), )?, - Span::new_styled(style(format!("{}", report.exit_code)).attribute(Attribute::Bold))?, + Span::new_styled( + style(format!("{}", report.cli_exit_code())).attribute(Attribute::Bold), + )?, ])), } Ok(output) diff --git a/cli/src/summary_output/upload.rs b/cli/src/summary_output/upload.rs index 667daa8e..43de90a7 100644 --- a/cli/src/summary_output/upload.rs +++ b/cli/src/summary_output/upload.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use api::urls::{url_for_test_case, url_for_upload}; use bundle::Test; -use constants::EXIT_SUCCESS; +use constants::{EXIT_FAILURE, EXIT_SUCCESS}; use context::junit::validator::JunitValidationLevel; use schemars::JsonSchema; use serde::Serialize; @@ -15,6 +15,8 @@ use crate::{upload_command::UploadRunResult, validate_command::JunitReportValida pub enum UploadCommand { Upload, Test, + /// The RSpec gem, which leaves the exit code to RSpec. + RSpec, } #[derive(Debug, Clone, Serialize, JsonSchema)] @@ -37,8 +39,9 @@ pub struct UploadReport { /// Failed tests that weren't quarantined, grouped by file, else by suite. pub failures: Vec, pub outcome: Outcome, - /// The code the CLI exits with. - pub exit_code: i32, + /// The code the CLI exits with. Left out by the RSpec gem, where RSpec decides the exit code. + #[serde(skip_serializing_if = "Option::is_none")] + pub exit_code: Option, /// Warnings that didn't stop the run, such as a deprecated flag. pub notices: Vec, } @@ -147,11 +150,14 @@ impl UploadReport { ); let error = result.error_report.as_ref().map(ErrorSummary::new); let exit_code = match command { - UploadCommand::Upload => error - .as_ref() - .and_then(|error| error.exit_code) - .unwrap_or(quarantine_context.exit_code), - UploadCommand::Test => quarantine_context.exit_code, + UploadCommand::Upload => Some( + error + .as_ref() + .and_then(|error| error.exit_code) + .unwrap_or(quarantine_context.exit_code), + ), + UploadCommand::Test => Some(quarantine_context.exit_code), + UploadCommand::RSpec => None, }; let bundle_upload_id = &result.meta.base_props.bundle_upload_id; @@ -172,6 +178,13 @@ impl UploadReport { } } +impl UploadReport { + /// What `upload` and `test` exit with; only the RSpec gem leaves `exit_code` unset. + pub fn cli_exit_code(&self) -> i32 { + self.exit_code.unwrap_or(EXIT_FAILURE) + } +} + impl GroupKey { fn of(test: &Test) -> Self { match &test.file { diff --git a/cli/tests/summary/schema.rs b/cli/tests/summary/schema.rs index ba522d7a..4606a1bc 100644 --- a/cli/tests/summary/schema.rs +++ b/cli/tests/summary/schema.rs @@ -1,6 +1,6 @@ use assert_cmd::{Command, cargo}; use pretty_assertions::assert_eq; -use trunk_analytics_cli::report::json; +use trunk_analytics_cli::summary_output::json; #[test] fn prints_the_summary_output_schema() { diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index 6935cf71..d35a09d7 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -835,9 +835,12 @@ fn upload_exits_with_the_exit_code_of_an_error_but_test_does_not() { assert_eq!( UploadReport::new(&result, UploadCommand::Upload).exit_code, - 70 + Some(70) + ); + assert_eq!( + UploadReport::new(&result, UploadCommand::Test).exit_code, + Some(1) ); - assert_eq!(UploadReport::new(&result, UploadCommand::Test).exit_code, 1); } #[test] @@ -852,9 +855,12 @@ fn exits_with_the_test_run_exit_code_when_an_error_has_none() { assert_eq!( UploadReport::new(&result, UploadCommand::Upload).exit_code, - 1 + Some(1) + ); + assert_eq!( + UploadReport::new(&result, UploadCommand::Test).exit_code, + Some(1) ); - assert_eq!(UploadReport::new(&result, UploadCommand::Test).exit_code, 1); } fn upload_json(result: &UploadRunResult) -> serde_json::Value { diff --git a/rspec-trunk-flaky-tests/README.md b/rspec-trunk-flaky-tests/README.md index 433e2068..ec18fd3f 100644 --- a/rspec-trunk-flaky-tests/README.md +++ b/rspec-trunk-flaky-tests/README.md @@ -118,6 +118,14 @@ By default (when unset or not `true`), quarantine lookup failures print a warnin TRUNK_QUARANTINE_QUERY_FAILURE_EXIT=true bundle exec rspec ``` +#### `TRUNK_SUMMARY_OUTPUT_FILE` + +When set to a file path, the gem writes a JSON summary of the run to that file after uploading: the test counts, and each failure with its link to Trunk. It's the same JSON that `trunk-analytics-cli upload --summary-output-file` writes, except that it has no `exitCode`, because RSpec decides the exit code. `trunk-analytics-cli summary-output-schema` prints its JSON Schema. Nothing is written when `TRUNK_LOCAL_UPLOAD_DIR` is set, since nothing is uploaded. + +```bash +TRUNK_SUMMARY_OUTPUT_FILE=tmp/trunk-summary.json bundle exec rspec +``` + #### `TRUNK_LOCAL_UPLOAD_DIR` (Experimental) > **⚠️ Experimental Feature**: This feature is experimental. Please reach out to support@trunk.io before attempting to use it. diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 2a67ffc5..d44d5511 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -27,6 +27,7 @@ # TRUNK_DRY_RUN - Set to 'true' to save bundle locally instead of uploading # TRUNK_USE_UNCLONED_REPO - Set to 'true' for uncloned repo mode # TRUNK_LOCAL_UPLOAD_DIR - Directory to save test results locally (disables upload) +# TRUNK_SUMMARY_OUTPUT_FILE - Path to write a JSON summary of the uploaded results to # TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS - Time to cache quarantined tests on disk (in seconds) # TRUNK_QUARANTINE_QUERY_FAILURE_EXIT - Set to 'true' to abort the RSpec run when quarantine # lookup fails (remaining examples are skipped) diff --git a/test_report/src/report.rs b/test_report/src/report.rs index f27b4796..3e348eaa 100644 --- a/test_report/src/report.rs +++ b/test_report/src/report.rs @@ -30,6 +30,12 @@ use third_party::sentry; use tracing_subscriber::{filter::FilterFn, prelude::*}; use trunk_analytics_cli::{ context::gather_initial_test_context, + error_report::ErrorReport, + summary_output::{ + ErrorSummary, UploadCommand, UploadReport, + json::{self, Report}, + text::SummaryFileText, + }, upload_command::{RunUploadOptions, run_upload}, }; use uuid::Uuid; @@ -818,6 +824,10 @@ impl MutTestReport { } let variant = env::var(constants::TRUNK_VARIANT_ENV).ok(); + let summary_output_file = env::var(constants::TRUNK_SUMMARY_OUTPUT_FILE_ENV) + .ok() + .filter(|path| !path.is_empty()) + .map(PathBuf::from); if let Some(ref variant) = variant { let test_report = &mut self.0.borrow_mut().test_report; @@ -956,6 +966,16 @@ impl MutTestReport { }, )) { Ok(upload_result) => { + if let Some(path) = &summary_output_file { + let mut report = + UploadReport::new(&upload_result, UploadCommand::RSpec); + // The upload trusts the gem's quarantine results, so only the gem knows + // whether its lookup failed. + report.quarantine_lookup_failed = + self.0.borrow().quarantine_lookup_source + == Some(QuarantineLookupSource::Failed); + write_summary_file(path, Report::Upload(&report)); + } if let Some(upload_bundle_error) = upload_result.error_report.map(|e| e.error) { @@ -969,13 +989,21 @@ impl MutTestReport { } } Err(e) => { - tracing::error!("Error uploading test report bundle: {:?}", e); + let report = ErrorReport::unlogged(e, self.get_org_url_slug(), None); + if let Some(path) = &summary_output_file { + write_summary_file(path, Report::Error(&ErrorSummary::new(&report))); + } + tracing::error!("Error uploading test report bundle: {:?}", report.error); false } } } Err(e) => { - tracing::error!("Error gathering initial context: {:?}", e); + let report = ErrorReport::unlogged(e, self.get_org_url_slug(), None); + if let Some(path) = &summary_output_file { + write_summary_file(path, Report::Error(&ErrorSummary::new(&report))); + } + tracing::error!("Error gathering initial context: {:?}", report.error); false } }; @@ -1105,6 +1133,14 @@ impl MutTestReport { } } +/// On stderr, as the CLI prints it: the gem's own logs only reach the console at error level. +fn write_summary_file(path: &Path, report: Report) { + eprintln!( + "{}", + SummaryFileText::new(path, json::write_file(path, report)).message() + ); +} + /// Microseconds are about all the precision an f64 epoch holds. fn timestamp_from_epoch_secs(secs: f64) -> Timestamp { DateTime::from_timestamp_micros((secs * 1_000_000.0).round() as i64) diff --git a/test_report/tests/common/env.rs b/test_report/tests/common/env.rs index 91e528eb..e4e1eb17 100644 --- a/test_report/tests/common/env.rs +++ b/test_report/tests/common/env.rs @@ -6,8 +6,9 @@ use constants::{ TRUNK_ORG_URL_SLUG_ENV, TRUNK_PR_NUMBER_ENV, TRUNK_PUBLIC_API_ADDRESS_ENV, TRUNK_QUARANTINE_DISK_CACHE_DIR_ENV, TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS_ENV, TRUNK_REPO_HEAD_AUTHOR_NAME_ENV, TRUNK_REPO_HEAD_BRANCH_ENV, TRUNK_REPO_HEAD_COMMIT_EPOCH_ENV, - TRUNK_REPO_HEAD_SHA_ENV, TRUNK_REPO_ROOT_ENV, TRUNK_REPO_URL_ENV, TRUNK_TEST_COLLECTION_ID_ENV, - TRUNK_USE_UNCLONED_REPO_ENV, TRUNK_VARIANT_ENV, + TRUNK_REPO_HEAD_SHA_ENV, TRUNK_REPO_ROOT_ENV, TRUNK_REPO_URL_ENV, + TRUNK_SUMMARY_OUTPUT_FILE_ENV, TRUNK_TEST_COLLECTION_ID_ENV, TRUNK_USE_UNCLONED_REPO_ENV, + TRUNK_VARIANT_ENV, }; use tempfile::TempDir; use test_report::report::quarantine_disk_cache_dir; @@ -37,6 +38,7 @@ pub fn cleanup_env_vars() { env::remove_var(TRUNK_API_CLIENT_RETRY_COUNT_ENV); env::remove_var(TRUNK_QUARANTINE_DISK_CACHE_DIR_ENV); env::remove_var(TRUNK_TEST_COLLECTION_ID_ENV); + env::remove_var(TRUNK_SUMMARY_OUTPUT_FILE_ENV); } } diff --git a/test_report/tests/publish.rs b/test_report/tests/publish.rs index 7d7bd738..74dc62d6 100644 --- a/test_report/tests/publish.rs +++ b/test_report/tests/publish.rs @@ -11,7 +11,9 @@ use std::{ thread, }; +use api::message::{GetQuarantineConfigRequest, GetQuarantineConfigResponse}; use assert_matches::assert_matches; +use axum::{Json, http::StatusCode}; use bundle::{BundleMeta, FileSetType}; use common::{cleanup_env_vars, setup_quarantine_disk_cache_dir}; use constants::{ @@ -19,7 +21,8 @@ use constants::{ TRUNK_DISABLE_QUARANTINING_ENV, TRUNK_DRY_RUN_ENV, TRUNK_ORG_URL_SLUG_ENV, TRUNK_PR_NUMBER_ENV, TRUNK_PUBLIC_API_ADDRESS_ENV, TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS_ENV, TRUNK_REPO_HEAD_AUTHOR_NAME_ENV, TRUNK_REPO_HEAD_BRANCH_ENV, TRUNK_REPO_HEAD_COMMIT_EPOCH_ENV, - TRUNK_REPO_HEAD_SHA_ENV, TRUNK_REPO_URL_ENV, TRUNK_USE_UNCLONED_REPO_ENV, TRUNK_VARIANT_ENV, + TRUNK_REPO_HEAD_SHA_ENV, TRUNK_REPO_ROOT_ENV, TRUNK_REPO_URL_ENV, + TRUNK_SUMMARY_OUTPUT_FILE_ENV, TRUNK_USE_UNCLONED_REPO_ENV, TRUNK_VARIANT_ENV, }; use context::repo::RepoUrlParts; use prost::Message; @@ -592,3 +595,146 @@ async fn publish_variant_priority_constructor_over_env() { let _ = env::set_current_dir(original_dir); } + +fn read_summary_output_file(path: &Path) -> serde_json::Value { + serde_json::from_reader(fs::File::open(path).unwrap()).unwrap() +} + +fn add_a_passing_and_a_failing_test(test_report: &MutTestReport) { + for (id, name, status) in [ + ("1", "passes", Status::Success), + ("2", "fails", Status::Failure), + ] { + test_report.add_test( + Some(id.into()), + name.into(), + "test-classname".into(), + "test-file".into(), + "test-parent-name".into(), + None, + status, + 0, + 1000.0, + 1001.0, + "test-message".into(), + "test-backtrace".into(), + false, + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn publish_writes_summary_output_file() { + cleanup_env_vars(); + let temp_dir = tempdir().unwrap(); + setup_quarantine_disk_cache_dir(&temp_dir); + setup_repo_with_commit(&temp_dir).unwrap(); + env::set_current_dir(&temp_dir).unwrap(); + let summary_path = temp_dir.path().join("summary.json"); + let state = MockServerBuilder::new().spawn_mock_server().await; + unsafe { + env::set_var(TRUNK_PUBLIC_API_ADDRESS_ENV, &state.host); + env::set_var("CI", "1"); + env::set_var("GITHUB_JOB", "test-job"); + env::set_var(TRUNK_API_TOKEN_ENV, "test-token"); + env::set_var(TRUNK_ORG_URL_SLUG_ENV, "test-org"); + env::set_var(TRUNK_SUMMARY_OUTPUT_FILE_ENV, &summary_path); + } + + thread::spawn(|| { + let test_report = MutTestReport::new("test".into(), "test-command 123".into(), None); + add_a_passing_and_a_failing_test(&test_report); + assert!(test_report.publish()); + }) + .join() + .unwrap(); + + let summary = read_summary_output_file(&summary_path); + assert_eq!(summary["schemaVersion"], 1); + assert_eq!(summary["type"], "upload"); + assert_eq!( + summary["counts"], + serde_json::json!({ "total": 2, "passed": 1, "failed": 1, "quarantined": 0, "passRatio": 0.5 }) + ); + assert_eq!(summary["outcome"], "failuresNotQuarantined"); + assert_eq!(summary["quarantineLookupFailed"], false); + assert_eq!(summary.get("exitCode"), None); + assert_eq!(summary["failures"][0]["tests"][0]["name"], "fails"); +} + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn publish_writes_a_failed_quarantine_lookup_to_summary_output_file() { + cleanup_env_vars(); + let temp_dir = tempdir().unwrap(); + setup_quarantine_disk_cache_dir(&temp_dir); + setup_repo_with_commit(&temp_dir).unwrap(); + env::set_current_dir(&temp_dir).unwrap(); + let summary_path = temp_dir.path().join("summary.json"); + let state = { + let mut builder = MockServerBuilder::new(); + builder.set_get_quarantining_config_handler( + |Json(_): Json| async { + Err::, StatusCode>(StatusCode::BAD_REQUEST) + }, + ); + builder.spawn_mock_server().await + }; + unsafe { + env::set_var(TRUNK_PUBLIC_API_ADDRESS_ENV, &state.host); + env::set_var(TRUNK_API_TOKEN_ENV, "test-token"); + env::set_var(TRUNK_ORG_URL_SLUG_ENV, "test-org"); + env::set_var(TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS_ENV, "0"); + env::set_var(TRUNK_SUMMARY_OUTPUT_FILE_ENV, &summary_path); + } + + thread::spawn(|| { + let test_report = MutTestReport::new("test".into(), "test-command 123".into(), None); + test_report.is_quarantined( + Some("2".into()), + Some("fails".into()), + Some("test-parent-name".into()), + Some("test-classname".into()), + Some("test-file".into()), + ); + add_a_passing_and_a_failing_test(&test_report); + assert!(test_report.publish()); + }) + .join() + .unwrap(); + + let summary = read_summary_output_file(&summary_path); + assert_eq!(summary["quarantineLookupFailed"], true); + assert_eq!(summary["counts"]["failed"], 1); +} + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn publish_writes_an_error_to_summary_output_file() { + cleanup_env_vars(); + let temp_dir = tempdir().unwrap(); + setup_quarantine_disk_cache_dir(&temp_dir); + env::set_current_dir(&temp_dir).unwrap(); + let summary_path = temp_dir.path().join("summary.json"); + let state = MockServerBuilder::new().spawn_mock_server().await; + unsafe { + env::set_var(TRUNK_PUBLIC_API_ADDRESS_ENV, &state.host); + env::set_var(TRUNK_API_TOKEN_ENV, "test-token"); + env::set_var(TRUNK_ORG_URL_SLUG_ENV, "test-org"); + env::set_var(TRUNK_REPO_ROOT_ENV, temp_dir.path()); + env::set_var(TRUNK_SUMMARY_OUTPUT_FILE_ENV, &summary_path); + } + + thread::spawn(|| { + let test_report = MutTestReport::new("test".into(), "test-command 123".into(), None); + assert!(!test_report.publish()); + }) + .join() + .unwrap(); + + let summary = read_summary_output_file(&summary_path); + assert_eq!(summary["schemaVersion"], 1); + assert_eq!(summary["type"], "error"); + assert_eq!(summary["kind"], "gitRepository"); +} From a234edb8d3cb146c0b4acebd90e56719f94142ac Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Fri, 23 Oct 2026 19:07:07 +0000 Subject: [PATCH 12/19] feat(cli): say in the summary whether it was a dry run The summary gets dryRun, from --dry-run (TRUNK_DRY_RUN in the RSpec gem). A dry run still checks quarantining but uploads nothing, so its summary has no upload ID or upload link and no error. dryRun says why. Refs TRUNK-22403 Co-Authored-By: Claude Opus 5.5 (1M context) --- cli/src/summary_output/upload.rs | 3 +++ cli/src/upload_command.rs | 2 ++ cli/tests/summary/upload.rs | 2 ++ cli/tests/upload.rs | 29 +++++++++++++++++++++++++++ test_report/tests/publish.rs | 34 ++++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+) diff --git a/cli/src/summary_output/upload.rs b/cli/src/summary_output/upload.rs index 43de90a7..7de66420 100644 --- a/cli/src/summary_output/upload.rs +++ b/cli/src/summary_output/upload.rs @@ -26,6 +26,8 @@ pub struct UploadReport { pub command: UploadCommand, /// Why uploading the results failed. The tests are still summarized. pub error: Option, + /// Nothing was uploaded (`--dry-run`), so there's no upload ID or upload link. + pub dry_run: bool, pub bundle_upload_id: Option, /// This upload's page in Trunk. Only set for uploads to a test collection. pub upload_url: Option, @@ -164,6 +166,7 @@ impl UploadReport { Self { command, error, + dry_run: result.dry_run, bundle_upload_id: Some(bundle_upload_id.clone()).filter(|id| !id.is_empty()), upload_url: upload_url(result), validations: file_validations(&result.validations), diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index a07ab776..7e369a03 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -486,6 +486,7 @@ pub struct UploadRunResult { pub api_address: String, pub guid_scope: Option, pub notices: Vec, + pub dry_run: bool, } pub struct RunUploadOptions { @@ -821,6 +822,7 @@ pub async fn run_upload( api_address: api_client.api_host.clone(), guid_scope, notices, + dry_run: upload_args.dry_run, }) } diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index d35a09d7..8e20402a 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -101,6 +101,7 @@ fn upload_result(num_tests: usize) -> UploadRunResult { api_address: String::from("https://api.trunk.io"), guid_scope: None, notices: Vec::new(), + dry_run: false, } } @@ -923,6 +924,7 @@ fn writes_the_report_as_json() { "settingsUrl": null, "exitCode": null, }, + "dryRun": false, "bundleUploadId": "4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", "uploadUrl": "https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", "validations": [ diff --git a/cli/tests/upload.rs b/cli/tests/upload.rs index 6a6bcf61..a5f924c9 100644 --- a/cli/tests/upload.rs +++ b/cli/tests/upload.rs @@ -1554,6 +1554,7 @@ async fn upload_writes_summary_output_file() { let summary = read_summary_output_file(&summary_path); assert_eq!(summary["schemaVersion"], 1); assert_eq!(summary["type"], "upload"); + assert_eq!(summary["dryRun"], false); assert_eq!( summary["exitCode"], assert.get_output().status.code().unwrap() @@ -1583,6 +1584,34 @@ async fn upload_writes_summary_output_file() { } } +#[tokio::test(flavor = "multi_thread")] +async fn upload_writes_a_dry_run_to_summary_output_file() { + let temp_dir = tempdir().unwrap(); + generate_mock_git_repo(&temp_dir); + generate_mock_valid_junit_xmls_with_failures(&temp_dir); + let summary_path = temp_dir.path().join("summary.json"); + + let state = MockServerBuilder::new().spawn_mock_server().await; + CommandBuilder::upload(temp_dir.path(), state.host.clone()) + .dry_run(true) + .command() + .arg("--summary-output-file") + .arg(&summary_path) + .assert() + .failure(); + + let requests = state.requests.lock().unwrap().clone(); + assert_eq!(requests.len(), 1); + assert_matches!(requests[0], RequestPayload::GetQuarantineConfig(..)); + + let summary = read_summary_output_file(&summary_path); + assert_eq!(summary["dryRun"], true); + assert_eq!(summary["error"], serde_json::Value::Null); + assert_eq!(summary["bundleUploadId"], serde_json::Value::Null); + assert_eq!(summary["uploadUrl"], serde_json::Value::Null); + assert!(summary["counts"]["failed"].as_u64().unwrap() > 0); +} + #[tokio::test(flavor = "multi_thread")] async fn upload_writes_an_error_to_summary_output_file() { let temp_dir = tempdir().unwrap(); diff --git a/test_report/tests/publish.rs b/test_report/tests/publish.rs index 74dc62d6..7746866f 100644 --- a/test_report/tests/publish.rs +++ b/test_report/tests/publish.rs @@ -659,10 +659,44 @@ async fn publish_writes_summary_output_file() { ); assert_eq!(summary["outcome"], "failuresNotQuarantined"); assert_eq!(summary["quarantineLookupFailed"], false); + assert_eq!(summary["dryRun"], false); assert_eq!(summary.get("exitCode"), None); assert_eq!(summary["failures"][0]["tests"][0]["name"], "fails"); } +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn publish_writes_a_dry_run_to_summary_output_file() { + cleanup_env_vars(); + let temp_dir = tempdir().unwrap(); + setup_quarantine_disk_cache_dir(&temp_dir); + setup_repo_with_commit(&temp_dir).unwrap(); + env::set_current_dir(&temp_dir).unwrap(); + let summary_path = temp_dir.path().join("summary.json"); + let state = MockServerBuilder::new().spawn_mock_server().await; + unsafe { + env::set_var(TRUNK_PUBLIC_API_ADDRESS_ENV, &state.host); + env::set_var(TRUNK_API_TOKEN_ENV, "test-token"); + env::set_var(TRUNK_ORG_URL_SLUG_ENV, "test-org"); + env::set_var(TRUNK_DRY_RUN_ENV, "true"); + env::set_var(TRUNK_SUMMARY_OUTPUT_FILE_ENV, &summary_path); + } + + thread::spawn(|| { + let test_report = MutTestReport::new("test".into(), "test-command 123".into(), None); + add_a_passing_and_a_failing_test(&test_report); + assert!(test_report.publish()); + }) + .join() + .unwrap(); + + assert!(state.requests.lock().unwrap().is_empty()); + let summary = read_summary_output_file(&summary_path); + assert_eq!(summary["dryRun"], true); + assert_eq!(summary["bundleUploadId"], serde_json::Value::Null); + assert_eq!(summary["counts"]["failed"], 1); +} + #[tokio::test(flavor = "multi_thread")] #[serial] async fn publish_writes_a_failed_quarantine_lookup_to_summary_output_file() { From 7b182b6ea510d060ceebeec055ce1733b3eddfc5 Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Fri, 23 Oct 2026 19:34:51 +0000 Subject: [PATCH 13/19] feat(cli): report the quarantining status and uppercase enum values `quarantining` replaces `quarantineLookupFailed` in the summary. Its value is one of: - CHECKED - NOT_NEEDED - DISABLED_VIA_CLI (--disable-quarantining) - DISABLED_VIA_API (quarantining is turned off for the repo in Trunk) - LOOKUP_FAILED So a reader can tell why no failure was quarantined. Telemetry's quarantine query result now comes from the same status. The RSpec gem takes the status from its own lookup, because it quarantines during the run even when TRUNK_DISABLE_QUARANTINING is set. Every enum value in the summary is now SCREAMING_SNAKE_CASE: `type`, `outcome`, the error and notice `kind`, and `quarantining`. Keys stay camelCase, as in the protobuf JSON mapping. A group's key becomes {"type": "FILE", "name": ...} or {"type": "OTHER"}, so its kind is a value rather than a key. Refs TRUNK-22403 Co-Authored-By: Claude Opus 5.5 (1M context) --- cli/src/context_quarantine.rs | 40 ++++--- cli/src/error_report.rs | 2 +- cli/src/summary_output/error.rs | 2 +- cli/src/summary_output/json.rs | 2 +- cli/src/summary_output/mod.rs | 4 +- cli/src/summary_output/notice.rs | 2 +- cli/src/summary_output/text.rs | 4 +- cli/src/summary_output/upload.rs | 28 ++++- cli/src/upload_command.rs | 2 + cli/tests/summary/error_report.rs | 6 +- cli/tests/summary/upload.rs | 42 +++++-- cli/tests/test.rs | 2 +- cli/tests/upload.rs | 28 ++++- test_report/src/report.rs | 21 +++- test_report/tests/publish.rs | 177 +++++++++++++++--------------- 15 files changed, 223 insertions(+), 139 deletions(-) diff --git a/cli/src/context_quarantine.rs b/cli/src/context_quarantine.rs index d3f5cab1..c10064bf 100644 --- a/cli/src/context_quarantine.rs +++ b/cli/src/context_quarantine.rs @@ -27,21 +27,14 @@ use pluralizer::pluralize; use prost::Message; use proto::upload_metrics::trunk::QuarantineQueryResult; +use crate::summary_output::Quarantining; + #[derive(Debug)] pub enum QuarantineFetchStatus { FetchSucceeded, FetchSkipped, FetchFailed(anyhow::Error), } -impl QuarantineFetchStatus { - pub fn is_failure(&self) -> bool { - match self { - Self::FetchSucceeded => false, - Self::FetchSkipped => false, - Self::FetchFailed(_) => true, - } - } -} #[derive(Debug)] pub struct QuarantineContext { @@ -54,6 +47,20 @@ pub struct QuarantineContext { pub quarantine_resolution_mode: QuarantineResolutionMode, } impl QuarantineContext { + pub fn quarantining(&self, disable_quarantining: bool) -> Quarantining { + if disable_quarantining { + Quarantining::DisabledViaCli + } else if self.quarantine_status.quarantining_disabled_for_repo { + Quarantining::DisabledViaApi + } else { + match &self.fetch_status { + QuarantineFetchStatus::FetchFailed(_) => Quarantining::LookupFailed, + QuarantineFetchStatus::FetchSkipped => Quarantining::NotNeeded, + QuarantineFetchStatus::FetchSucceeded => Quarantining::Checked, + } + } + } + pub fn skip_fetch(failures: Vec) -> Self { Self { exit_code: i32::default(), @@ -83,14 +90,13 @@ pub fn quarantine_query_result( disable_quarantining: bool, ctx: &QuarantineContext, ) -> proto::upload_metrics::trunk::QuarantineQueryResult { - if disable_quarantining || ctx.quarantine_status.quarantining_disabled_for_repo { - return QuarantineQueryResult::Disabled; - } - - match &ctx.fetch_status { - QuarantineFetchStatus::FetchFailed(_) => QuarantineQueryResult::Failure, - QuarantineFetchStatus::FetchSkipped => QuarantineQueryResult::Skipped, - QuarantineFetchStatus::FetchSucceeded => QuarantineQueryResult::Success, + match ctx.quarantining(disable_quarantining) { + Quarantining::DisabledViaCli | Quarantining::DisabledViaApi => { + QuarantineQueryResult::Disabled + } + Quarantining::LookupFailed => QuarantineQueryResult::Failure, + Quarantining::NotNeeded => QuarantineQueryResult::Skipped, + Quarantining::Checked => QuarantineQueryResult::Success, } } diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index 140ad170..ae4b3dc5 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -93,7 +93,7 @@ impl ErrorReport { } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum ErrorKind { /// Couldn't connect to Trunk. ConnectionRefused, diff --git a/cli/src/summary_output/error.rs b/cli/src/summary_output/error.rs index c0a3d91a..22790f88 100644 --- a/cli/src/summary_output/error.rs +++ b/cli/src/summary_output/error.rs @@ -16,7 +16,7 @@ pub struct ErrorSummary { pub causes: Vec, /// Trunk's explanation, when its API gave one. pub explanation: Option, - /// Where to check the org slug and token. Only set for `unauthorized` errors. + /// Where to check the org slug and token. Only set for `UNAUTHORIZED` errors. pub settings_url: Option, /// A run that fails outright exits with this code, or 0 when it's null. Upload reports exit with their own `exitCode`. pub exit_code: Option, diff --git a/cli/src/summary_output/json.rs b/cli/src/summary_output/json.rs index c5da3008..04ee8b7b 100644 --- a/cli/src/summary_output/json.rs +++ b/cli/src/summary_output/json.rs @@ -8,7 +8,7 @@ use super::{ErrorSummary, UploadReport}; pub const SCHEMA_VERSION: u32 = 1; #[derive(Debug, Clone, Copy, Serialize, JsonSchema)] -#[serde(tag = "type", rename_all = "camelCase")] +#[serde(tag = "type", rename_all = "SCREAMING_SNAKE_CASE")] pub enum Report<'a> { /// The run finished and summarized its tests. Upload(&'a UploadReport), diff --git a/cli/src/summary_output/mod.rs b/cli/src/summary_output/mod.rs index 0a4c899e..85e6b833 100644 --- a/cli/src/summary_output/mod.rs +++ b/cli/src/summary_output/mod.rs @@ -8,7 +8,7 @@ mod validate; pub use error::ErrorSummary; pub use notice::{Notice, NoticeKind}; pub use upload::{ - FailedTest, FailureGroup, FileValidation, GroupKey, Outcome, ParseError, TestCounts, - UploadCommand, UploadReport, + FailedTest, FailureGroup, FileValidation, GroupKey, Outcome, ParseError, Quarantining, + TestCounts, UploadCommand, UploadReport, }; pub use validate::{Codeowners, MatchedFiles, ValidateReport, ValidationCounts}; diff --git a/cli/src/summary_output/notice.rs b/cli/src/summary_output/notice.rs index c5a02848..2da04e21 100644 --- a/cli/src/summary_output/notice.rs +++ b/cli/src/summary_output/notice.rs @@ -9,7 +9,7 @@ pub struct Notice { } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum NoticeKind { /// A deprecated flag was passed. Deprecation, diff --git a/cli/src/summary_output/text.rs b/cli/src/summary_output/text.rs index e1f41ea8..f97c2dea 100644 --- a/cli/src/summary_output/text.rs +++ b/cli/src/summary_output/text.rs @@ -10,7 +10,7 @@ use superconsole::{ use super::{ ErrorSummary, FailureGroup, FileValidation, GroupKey, NoticeKind, Outcome, ParseError, - TestCounts, UploadReport, ValidateReport, ValidationCounts, + Quarantining, TestCounts, UploadReport, ValidateReport, ValidationCounts, }; use crate::{ error_report::{CONNECTION_REFUSED_CONTEXT, ErrorKind, HELP_TEXT, UNAUTHORIZED_CONTEXT}, @@ -214,7 +214,7 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re " Total: {} Pass: {} Fail: {} Quarantined: {} Pass Ratio: {}", total, passed, failed, quarantined, pass_ratio ))?])); - if report.quarantine_lookup_failed { + if report.quarantining == Quarantining::LookupFailed { output.push(Line::from_iter([Span::new_styled( style(" We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures".to_string()).with(Color::Yellow).attribute(Attribute::Bold), )?])); diff --git a/cli/src/summary_output/upload.rs b/cli/src/summary_output/upload.rs index 7de66420..58e55015 100644 --- a/cli/src/summary_output/upload.rs +++ b/cli/src/summary_output/upload.rs @@ -11,11 +11,12 @@ use super::{ErrorSummary, Notice}; use crate::{upload_command::UploadRunResult, validate_command::JunitReportValidations}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum UploadCommand { Upload, Test, /// The RSpec gem, which leaves the exit code to RSpec. + #[serde(rename = "RSPEC")] RSpec, } @@ -34,8 +35,7 @@ pub struct UploadReport { /// One entry per test report file. pub validations: Vec, pub counts: TestCounts, - /// Trunk couldn't say which tests are quarantined, so no failed test counts as quarantined. - pub quarantine_lookup_failed: bool, + pub quarantining: Quarantining, /// Failed tests that were quarantined, grouped like `failures`. pub quarantined: Vec, /// Failed tests that weren't quarantined, grouped by file, else by suite. @@ -71,7 +71,7 @@ pub struct FailureGroup { /// Groups sort by file, then by suite, then the rest, in declaration order. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] +#[serde(tag = "type", content = "name", rename_all = "SCREAMING_SNAKE_CASE")] pub enum GroupKey { /// Failed tests from this file. File(String), @@ -123,8 +123,24 @@ pub struct ParseError { pub causes: Vec, } +/// Whether the failed tests could be checked for quarantining, and if not, why. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum Quarantining { + /// Trunk said which failed tests are quarantined. + Checked, + /// Nothing needed looking up: no test failed, or the results already said which are quarantined. + NotNeeded, + /// Turned off for this run with `--disable-quarantining`. + DisabledViaCli, + /// Turned off for the repo in Trunk. + DisabledViaApi, + /// Trunk couldn't say which tests are quarantined, so no failed test counts as quarantined. + LookupFailed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum Outcome { /// The test reports had no tests. NoTestsFound, @@ -171,7 +187,7 @@ impl UploadReport { upload_url: upload_url(result), validations: file_validations(&result.validations), counts, - quarantine_lookup_failed: quarantine_context.fetch_status.is_failure(), + quarantining: quarantine_context.quarantining(result.disable_quarantining), quarantined: group(quarantined, |test| failed_test(result, test, true)), failures: group(failures, |test| failed_test(result, test, false)), outcome: Outcome::new(&counts, quarantine_context.exit_code), diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index 7e369a03..0f85ebaa 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -487,6 +487,7 @@ pub struct UploadRunResult { pub guid_scope: Option, pub notices: Vec, pub dry_run: bool, + pub disable_quarantining: bool, } pub struct RunUploadOptions { @@ -823,6 +824,7 @@ pub async fn run_upload( guid_scope, notices, dry_run: upload_args.dry_run, + disable_quarantining, }) } diff --git a/cli/tests/summary/error_report.rs b/cli/tests/summary/error_report.rs index 5d7c1c07..7e155f7e 100644 --- a/cli/tests/summary/error_report.rs +++ b/cli/tests/summary/error_report.rs @@ -202,8 +202,8 @@ fn writes_an_error_as_json() { .unwrap(), serde_json::json!({ "schemaVersion": 1, - "type": "error", - "kind": "other", + "type": "ERROR", + "kind": "OTHER", "message": null, "error": "Failed to gather upload context", "causes": ["--allow-forked-pr-uploads requires --test-collection-id"], @@ -221,7 +221,7 @@ fn writes_the_settings_url_and_exit_code_of_an_unauthorized_error_as_json() { let json: serde_json::Value = serde_json::from_str(&json::to_string(Report::Error(&error)).unwrap()).unwrap(); - assert_eq!(json["kind"], "unauthorized"); + assert_eq!(json["kind"], "UNAUTHORIZED"); assert_eq!( json["settingsUrl"], "https://app.trunk.io/fake-org-slug/settings" diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index 8e20402a..de2770c0 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -102,6 +102,7 @@ fn upload_result(num_tests: usize) -> UploadRunResult { guid_scope: None, notices: Vec::new(), dry_run: false, + disable_quarantining: false, } } @@ -914,9 +915,9 @@ fn writes_the_report_as_json() { upload_json(&result), serde_json::json!({ "schemaVersion": 1, - "type": "upload", + "type": "UPLOAD", "error": { - "kind": "other", + "kind": "OTHER", "message": "There was an unexpected error that occurred while uploading test results", "error": "Failed to create bundle upload", "causes": ["HTTP status server error (500 Internal Server Error)"], @@ -955,10 +956,10 @@ fn writes_the_report_as_json() { "quarantined": 1, "passRatio": 0.7, }, - "quarantineLookupFailed": false, + "quarantining": "CHECKED", "quarantined": [ { - "key": { "file": "src/math.rs" }, + "key": { "type": "FILE", "name": "src/math.rs" }, "tests": [ { "name": "adds", @@ -976,7 +977,7 @@ fn writes_the_report_as_json() { ], "failures": [ { - "key": { "file": "src/math.rs" }, + "key": { "type": "FILE", "name": "src/math.rs" }, "tests": [ { "name": "subtracts", @@ -992,7 +993,7 @@ fn writes_the_report_as_json() { ], }, { - "key": "other", + "key": { "type": "OTHER" }, "tests": [ { "name": "boots", @@ -1008,14 +1009,39 @@ fn writes_the_report_as_json() { ], }, ], - "outcome": "failuresNotQuarantined", + "outcome": "FAILURES_NOT_QUARANTINED", "exitCode": 1, "notices": [ { - "kind": "deprecation", + "kind": "DEPRECATION", "message": "The --team flag is deprecated and will be removed in a future version.", }, ], }) ); } + +#[test] +fn says_whether_failures_could_be_checked_for_quarantining() { + let quarantining = |result: &UploadRunResult| { + serde_json::to_value(UploadReport::new(result, UploadCommand::Upload).quarantining).unwrap() + }; + let mut result = upload_result(1); + assert_eq!(quarantining(&result), "NOT_NEEDED"); + + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + assert_eq!(quarantining(&result), "CHECKED"); + + result.quarantine_context.fetch_status = + QuarantineFetchStatus::FetchFailed(anyhow::anyhow!("request timed out")); + assert_eq!(quarantining(&result), "LOOKUP_FAILED"); + + result + .quarantine_context + .quarantine_status + .quarantining_disabled_for_repo = true; + assert_eq!(quarantining(&result), "DISABLED_VIA_API"); + + result.disable_quarantining = true; + assert_eq!(quarantining(&result), "DISABLED_VIA_CLI"); +} diff --git a/cli/tests/test.rs b/cli/tests/test.rs index 9c43e508..c8277514 100644 --- a/cli/tests/test.rs +++ b/cli/tests/test.rs @@ -584,6 +584,6 @@ async fn test_command_writes_summary_output_file() { let summary: serde_json::Value = serde_json::from_reader(File::open(&summary_path).unwrap()).unwrap(); assert_eq!(summary["schemaVersion"], 1); - assert_eq!(summary["type"], "upload"); + assert_eq!(summary["type"], "UPLOAD"); assert_eq!(summary["exitCode"], 1); } diff --git a/cli/tests/upload.rs b/cli/tests/upload.rs index a5f924c9..14f59427 100644 --- a/cli/tests/upload.rs +++ b/cli/tests/upload.rs @@ -1553,8 +1553,9 @@ async fn upload_writes_summary_output_file() { let summary = read_summary_output_file(&summary_path); assert_eq!(summary["schemaVersion"], 1); - assert_eq!(summary["type"], "upload"); + assert_eq!(summary["type"], "UPLOAD"); assert_eq!(summary["dryRun"], false); + assert_eq!(summary["quarantining"], "CHECKED"); assert_eq!( summary["exitCode"], assert.get_output().status.code().unwrap() @@ -1606,12 +1607,33 @@ async fn upload_writes_a_dry_run_to_summary_output_file() { let summary = read_summary_output_file(&summary_path); assert_eq!(summary["dryRun"], true); + assert_eq!(summary["quarantining"], "CHECKED"); assert_eq!(summary["error"], serde_json::Value::Null); assert_eq!(summary["bundleUploadId"], serde_json::Value::Null); assert_eq!(summary["uploadUrl"], serde_json::Value::Null); assert!(summary["counts"]["failed"].as_u64().unwrap() > 0); } +#[tokio::test(flavor = "multi_thread")] +async fn upload_writes_that_quarantining_was_disabled_to_summary_output_file() { + let temp_dir = tempdir().unwrap(); + generate_mock_git_repo(&temp_dir); + generate_mock_valid_junit_xmls_with_failures(&temp_dir); + let summary_path = temp_dir.path().join("summary.json"); + + let state = MockServerBuilder::new().spawn_mock_server().await; + CommandBuilder::upload(temp_dir.path(), state.host.clone()) + .disable_quarantining(true) + .command() + .arg("--summary-output-file") + .arg(&summary_path) + .assert() + .success(); + + let summary = read_summary_output_file(&summary_path); + assert_eq!(summary["quarantining"], "DISABLED_VIA_CLI"); +} + #[tokio::test(flavor = "multi_thread")] async fn upload_writes_an_error_to_summary_output_file() { let temp_dir = tempdir().unwrap(); @@ -1629,8 +1651,8 @@ async fn upload_writes_an_error_to_summary_output_file() { let summary = read_summary_output_file(&summary_path); assert_eq!(summary["schemaVersion"], 1); - assert_eq!(summary["type"], "error"); - assert_eq!(summary["kind"], "gitRepository"); + assert_eq!(summary["type"], "ERROR"); + assert_eq!(summary["kind"], "GIT_REPOSITORY"); assert_eq!(summary["exitCode"], exitcode::SOFTWARE); } diff --git a/test_report/src/report.rs b/test_report/src/report.rs index 3e348eaa..75b7b022 100644 --- a/test_report/src/report.rs +++ b/test_report/src/report.rs @@ -32,7 +32,7 @@ use trunk_analytics_cli::{ context::gather_initial_test_context, error_report::ErrorReport, summary_output::{ - ErrorSummary, UploadCommand, UploadReport, + ErrorSummary, Quarantining, UploadCommand, UploadReport, json::{self, Report}, text::SummaryFileText, }, @@ -353,6 +353,18 @@ impl MutTestReport { } } + /// `TRUNK_DISABLE_QUARANTINING` isn't here because the gem quarantines during the run regardless. + fn quarantining(&self) -> Quarantining { + match self.0.borrow().quarantine_lookup_source { + Some(QuarantineLookupSource::Api | QuarantineLookupSource::DiskCache) => { + Quarantining::Checked + } + Some(QuarantineLookupSource::Disabled) => Quarantining::DisabledViaApi, + Some(QuarantineLookupSource::Failed) => Quarantining::LookupFailed, + None => Quarantining::NotNeeded, + } + } + fn quarantine_resolution_mode(&self) -> QuarantineResolutionMode { self.0 .borrow() @@ -969,11 +981,8 @@ impl MutTestReport { if let Some(path) = &summary_output_file { let mut report = UploadReport::new(&upload_result, UploadCommand::RSpec); - // The upload trusts the gem's quarantine results, so only the gem knows - // whether its lookup failed. - report.quarantine_lookup_failed = - self.0.borrow().quarantine_lookup_source - == Some(QuarantineLookupSource::Failed); + // The upload trusts the gem's quarantine results without looking them up. + report.quarantining = self.quarantining(); write_summary_file(path, Report::Upload(&report)); } if let Some(upload_bundle_error) = diff --git a/test_report/tests/publish.rs b/test_report/tests/publish.rs index 7746866f..f50f2399 100644 --- a/test_report/tests/publish.rs +++ b/test_report/tests/publish.rs @@ -3,7 +3,7 @@ mod common; use std::{ env, fs, io::BufReader, - path::Path, + path::{Path, PathBuf}, sync::{ Arc, atomic::{AtomicBool, Ordering}, @@ -29,7 +29,7 @@ use prost::Message; use prost_wkt_types::Timestamp; use proto::test_context::test_run::{AttemptNumber, TestCaseRunStatus, TestOutput, TestReport}; use serial_test::serial; -use tempfile::tempdir; +use tempfile::{TempDir, tempdir}; use test_report::report::{MutTestReport, Status}; use test_utils::mock_git_repo::setup_repo_with_commit; use test_utils::mock_server::{MockServerBuilder, RequestPayload}; @@ -600,65 +600,78 @@ fn read_summary_output_file(path: &Path) -> serde_json::Value { serde_json::from_reader(fs::File::open(path).unwrap()).unwrap() } -fn add_a_passing_and_a_failing_test(test_report: &MutTestReport) { - for (id, name, status) in [ - ("1", "passes", Status::Success), - ("2", "fails", Status::Failure), - ] { - test_report.add_test( - Some(id.into()), - name.into(), - "test-classname".into(), - "test-file".into(), - "test-parent-name".into(), - None, - status, - 0, - 1000.0, - 1001.0, - "test-message".into(), - "test-backtrace".into(), - false, - ); - } -} - -#[tokio::test(flavor = "multi_thread")] -#[serial] -async fn publish_writes_summary_output_file() { - cleanup_env_vars(); - let temp_dir = tempdir().unwrap(); - setup_quarantine_disk_cache_dir(&temp_dir); - setup_repo_with_commit(&temp_dir).unwrap(); - env::set_current_dir(&temp_dir).unwrap(); +fn set_up_summary_output_file(temp_dir: &TempDir, host: &str) -> PathBuf { + setup_quarantine_disk_cache_dir(temp_dir); + setup_repo_with_commit(temp_dir).unwrap(); + env::set_current_dir(temp_dir).unwrap(); let summary_path = temp_dir.path().join("summary.json"); - let state = MockServerBuilder::new().spawn_mock_server().await; unsafe { - env::set_var(TRUNK_PUBLIC_API_ADDRESS_ENV, &state.host); - env::set_var("CI", "1"); - env::set_var("GITHUB_JOB", "test-job"); + env::set_var(TRUNK_PUBLIC_API_ADDRESS_ENV, host); env::set_var(TRUNK_API_TOKEN_ENV, "test-token"); env::set_var(TRUNK_ORG_URL_SLUG_ENV, "test-org"); + env::set_var(TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS_ENV, "0"); env::set_var(TRUNK_SUMMARY_OUTPUT_FILE_ENV, &summary_path); } + summary_path +} - thread::spawn(|| { +fn publish_a_passing_and_a_failing_test(check_quarantine: bool) -> bool { + thread::spawn(move || { let test_report = MutTestReport::new("test".into(), "test-command 123".into(), None); - add_a_passing_and_a_failing_test(&test_report); - assert!(test_report.publish()); + if check_quarantine { + test_report.is_quarantined( + Some("2".into()), + Some("fails".into()), + Some("test-parent-name".into()), + Some("test-classname".into()), + Some("test-file".into()), + ); + } + for (id, name, status) in [ + ("1", "passes", Status::Success), + ("2", "fails", Status::Failure), + ] { + test_report.add_test( + Some(id.into()), + name.into(), + "test-classname".into(), + "test-file".into(), + "test-parent-name".into(), + None, + status, + 0, + 1000.0, + 1001.0, + "test-message".into(), + "test-backtrace".into(), + false, + ); + } + test_report.publish() }) .join() - .unwrap(); + .unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn publish_writes_summary_output_file() { + cleanup_env_vars(); + let temp_dir = tempdir().unwrap(); + let state = MockServerBuilder::new().spawn_mock_server().await; + let summary_path = set_up_summary_output_file(&temp_dir, &state.host); + + assert!(publish_a_passing_and_a_failing_test(true)); let summary = read_summary_output_file(&summary_path); assert_eq!(summary["schemaVersion"], 1); - assert_eq!(summary["type"], "upload"); + assert_eq!(summary["type"], "UPLOAD"); assert_eq!( summary["counts"], serde_json::json!({ "total": 2, "passed": 1, "failed": 1, "quarantined": 0, "passRatio": 0.5 }) ); - assert_eq!(summary["outcome"], "failuresNotQuarantined"); - assert_eq!(summary["quarantineLookupFailed"], false); + assert_eq!(summary["outcome"], "FAILURES_NOT_QUARANTINED"); + assert_eq!(summary["quarantining"], "CHECKED"); assert_eq!(summary["dryRun"], false); assert_eq!(summary.get("exitCode"), None); assert_eq!(summary["failures"][0]["tests"][0]["name"], "fails"); @@ -669,30 +682,18 @@ async fn publish_writes_summary_output_file() { async fn publish_writes_a_dry_run_to_summary_output_file() { cleanup_env_vars(); let temp_dir = tempdir().unwrap(); - setup_quarantine_disk_cache_dir(&temp_dir); - setup_repo_with_commit(&temp_dir).unwrap(); - env::set_current_dir(&temp_dir).unwrap(); - let summary_path = temp_dir.path().join("summary.json"); let state = MockServerBuilder::new().spawn_mock_server().await; + let summary_path = set_up_summary_output_file(&temp_dir, &state.host); unsafe { - env::set_var(TRUNK_PUBLIC_API_ADDRESS_ENV, &state.host); - env::set_var(TRUNK_API_TOKEN_ENV, "test-token"); - env::set_var(TRUNK_ORG_URL_SLUG_ENV, "test-org"); env::set_var(TRUNK_DRY_RUN_ENV, "true"); - env::set_var(TRUNK_SUMMARY_OUTPUT_FILE_ENV, &summary_path); } - thread::spawn(|| { - let test_report = MutTestReport::new("test".into(), "test-command 123".into(), None); - add_a_passing_and_a_failing_test(&test_report); - assert!(test_report.publish()); - }) - .join() - .unwrap(); + assert!(publish_a_passing_and_a_failing_test(false)); assert!(state.requests.lock().unwrap().is_empty()); let summary = read_summary_output_file(&summary_path); assert_eq!(summary["dryRun"], true); + assert_eq!(summary["quarantining"], "NOT_NEEDED"); assert_eq!(summary["bundleUploadId"], serde_json::Value::Null); assert_eq!(summary["counts"]["failed"], 1); } @@ -702,10 +703,6 @@ async fn publish_writes_a_dry_run_to_summary_output_file() { async fn publish_writes_a_failed_quarantine_lookup_to_summary_output_file() { cleanup_env_vars(); let temp_dir = tempdir().unwrap(); - setup_quarantine_disk_cache_dir(&temp_dir); - setup_repo_with_commit(&temp_dir).unwrap(); - env::set_current_dir(&temp_dir).unwrap(); - let summary_path = temp_dir.path().join("summary.json"); let state = { let mut builder = MockServerBuilder::new(); builder.set_get_quarantining_config_handler( @@ -715,34 +712,40 @@ async fn publish_writes_a_failed_quarantine_lookup_to_summary_output_file() { ); builder.spawn_mock_server().await }; - unsafe { - env::set_var(TRUNK_PUBLIC_API_ADDRESS_ENV, &state.host); - env::set_var(TRUNK_API_TOKEN_ENV, "test-token"); - env::set_var(TRUNK_ORG_URL_SLUG_ENV, "test-org"); - env::set_var(TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS_ENV, "0"); - env::set_var(TRUNK_SUMMARY_OUTPUT_FILE_ENV, &summary_path); - } + let summary_path = set_up_summary_output_file(&temp_dir, &state.host); - thread::spawn(|| { - let test_report = MutTestReport::new("test".into(), "test-command 123".into(), None); - test_report.is_quarantined( - Some("2".into()), - Some("fails".into()), - Some("test-parent-name".into()), - Some("test-classname".into()), - Some("test-file".into()), - ); - add_a_passing_and_a_failing_test(&test_report); - assert!(test_report.publish()); - }) - .join() - .unwrap(); + assert!(publish_a_passing_and_a_failing_test(true)); let summary = read_summary_output_file(&summary_path); - assert_eq!(summary["quarantineLookupFailed"], true); + assert_eq!(summary["quarantining"], "LOOKUP_FAILED"); assert_eq!(summary["counts"]["failed"], 1); } +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn publish_writes_that_quarantining_is_disabled_for_the_repo_to_summary_output_file() { + cleanup_env_vars(); + let temp_dir = tempdir().unwrap(); + let state = { + let mut builder = MockServerBuilder::new(); + builder.set_get_quarantining_config_handler( + |Json(_): Json| async { + Json(GetQuarantineConfigResponse { + is_disabled: true, + ..Default::default() + }) + }, + ); + builder.spawn_mock_server().await + }; + let summary_path = set_up_summary_output_file(&temp_dir, &state.host); + + assert!(publish_a_passing_and_a_failing_test(true)); + + let summary = read_summary_output_file(&summary_path); + assert_eq!(summary["quarantining"], "DISABLED_VIA_API"); +} + #[tokio::test(flavor = "multi_thread")] #[serial] async fn publish_writes_an_error_to_summary_output_file() { @@ -769,6 +772,6 @@ async fn publish_writes_an_error_to_summary_output_file() { let summary = read_summary_output_file(&summary_path); assert_eq!(summary["schemaVersion"], 1); - assert_eq!(summary["type"], "error"); - assert_eq!(summary["kind"], "gitRepository"); + assert_eq!(summary["type"], "ERROR"); + assert_eq!(summary["kind"], "GIT_REPOSITORY"); } From ad00eed7d72b73a8174305a2b647a908e2af8a36 Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 28 Oct 2026 16:09:27 +0000 Subject: [PATCH 14/19] test(cli): cover the command and failure text in the JSON summary Co-Authored-By: Claude Opus 5.5 --- cli/tests/summary/upload.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index de2770c0..4e3e1032 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -901,7 +901,8 @@ fn writes_the_report_as_json() { result.quarantine_context.failures = vec![ Test { class_name: Some(String::from("Math")), - failure_message: Some(String::from("expected 1, got 2")), + failure_message_attribute: Some(String::from("expected 1, got 2")), + failure_text: Some(String::from("at src/math.rs:12")), variant: Some(String::from("linux")), ..failure("subtracts", "MathTests", Some("src/math.rs")) }, @@ -916,6 +917,7 @@ fn writes_the_report_as_json() { serde_json::json!({ "schemaVersion": 1, "type": "UPLOAD", + "command": "UPLOAD", "error": { "kind": "OTHER", "message": "There was an unexpected error that occurred while uploading test results", @@ -969,6 +971,7 @@ fn writes_the_report_as_json() { "id": "adds-id", "variant": null, "failureMessage": null, + "failureText": null, "quarantined": true, "link": "https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/adds-id?repo=trunk-io%2Fanalytics-cli", }, @@ -987,6 +990,7 @@ fn writes_the_report_as_json() { "id": "subtracts-id", "variant": "linux", "failureMessage": "expected 1, got 2", + "failureText": "at src/math.rs:12", "quarantined": false, "link": "https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/subtracts-id?repo=trunk-io%2Fanalytics-cli", }, @@ -1003,6 +1007,7 @@ fn writes_the_report_as_json() { "id": "boots-id", "variant": null, "failureMessage": null, + "failureText": null, "quarantined": false, "link": "https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/boots-id?repo=trunk-io%2Fanalytics-cli", }, From d3aa7efc86da8b3d31928845a87436bc77a20407 Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Sat, 24 Oct 2026 00:11:28 +0000 Subject: [PATCH 15/19] refactor(cli): define each exit code in one place, with its documentation Add `ExitCode` (cli/src/exit_code.rs) with a variant per code and a doc comment on each saying when it's used, plus `PassedThrough(code)` for the test command's code and --test-process-exit-code. Two exit codes are equal when the process exits with the same number. Every exit code decision and every field that holds one now uses it: quarantining, ErrorKind and ErrorReport, UploadReport (including cli_exit_code()), ValidateReport, ErrorSummary, the test command, main.rs and shutdown.rs. It serializes as a plain number, so the JSON doesn't change. The doc comments generate the user-facing docs: - an "Exit codes" section in --help (via clap's after_long_help), on the top-level command and each subcommand; - the ExitCode description in summary-output-schema. Remove EXIT_SUCCESS and EXIT_FAILURE from constants and drop the exitcode dependency. No behavior change: the tests only change how they build their inputs and name their expected codes; the expected numbers and text are the same. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 7 - cli/Cargo.toml | 1 - cli/src/context.rs | 3 +- cli/src/context_quarantine.rs | 26 ++-- cli/src/error_report.rs | 12 +- cli/src/exit_code.rs | 225 +++++++++++++++++++++++++++++ cli/src/lib.rs | 1 + cli/src/main.rs | 15 +- cli/src/shutdown.rs | 15 +- cli/src/summary_output/error.rs | 7 +- cli/src/summary_output/text.rs | 8 +- cli/src/summary_output/upload.rs | 15 +- cli/src/summary_output/validate.rs | 12 +- cli/src/test_command.rs | 6 +- cli/src/upload_command.rs | 9 +- cli/tests/summary/error_report.rs | 5 +- cli/tests/summary/upload.rs | 37 ++--- cli/tests/upload.rs | 8 +- constants/src/lib.rs | 3 - 19 files changed, 321 insertions(+), 94 deletions(-) create mode 100644 cli/src/exit_code.rs diff --git a/Cargo.lock b/Cargo.lock index 68015815..7469eef7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1328,12 +1328,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "exitcode" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" - [[package]] name = "fake" version = "2.10.0" @@ -5953,7 +5947,6 @@ dependencies = [ "constants", "context", "display", - "exitcode", "github-actions", "gix", "glob", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 8667c4c2..e3d30154 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -24,7 +24,6 @@ context = { path = "../context", features = ["bindings"] } third-party = { path = "../third-party" } display = { path = "../display" } log = "0.4.14" -exitcode = "1.1.1" gix = { version = "0.74.0", default-features = false, features = [ ], optional = true } http = "1.1.0" diff --git a/cli/src/context.rs b/cli/src/context.rs index cc0da916..0e16ab01 100644 --- a/cli/src/context.rs +++ b/cli/src/context.rs @@ -44,6 +44,7 @@ use xcresult::test_locations::{Limits, TestKey, TestLocationIndex}; use xcresult::xcresult::XCResult; use crate::error_report::InterruptingError; +use crate::exit_code::ExitCode; use crate::{ context_quarantine::{ FailedTestsExtractor, QuarantineContext, QuarantineFetchStatus, gather_quarantine_context, @@ -811,7 +812,7 @@ pub async fn gather_exit_code_and_quarantined_tests_context( disable_quarantining: bool, api_client: &ApiClient, file_set_builder: &FileSetBuilder, - default_exit_code: Option, + default_exit_code: Option, test_collection_short_id: Option, hide_test_collection_links: bool, ) -> anyhow::Result { diff --git a/cli/src/context_quarantine.rs b/cli/src/context_quarantine.rs index c10064bf..ea439399 100644 --- a/cli/src/context_quarantine.rs +++ b/cli/src/context_quarantine.rs @@ -12,7 +12,6 @@ use bundle::{ FileSet, FileSetBuilder, FileSetTestRunnerReport, FileSetType, QuarantineBulkTestStatus, Test, }; use chrono::TimeDelta; -use constants::{EXIT_FAILURE, EXIT_SUCCESS}; use context::{ junit::{ bindings::{ @@ -27,6 +26,7 @@ use pluralizer::pluralize; use prost::Message; use proto::upload_metrics::trunk::QuarantineQueryResult; +use crate::exit_code::ExitCode; use crate::summary_output::Quarantining; #[derive(Debug)] @@ -38,7 +38,7 @@ pub enum QuarantineFetchStatus { #[derive(Debug)] pub struct QuarantineContext { - pub exit_code: i32, + pub exit_code: ExitCode, pub quarantine_status: QuarantineBulkTestStatus, pub failures: Vec, pub repo: RepoUrlParts, @@ -63,7 +63,7 @@ impl QuarantineContext { pub fn skip_fetch(failures: Vec) -> Self { Self { - exit_code: i32::default(), + exit_code: ExitCode::Success, quarantine_status: QuarantineBulkTestStatus::default(), failures, repo: RepoUrlParts::default(), @@ -75,7 +75,7 @@ impl QuarantineContext { pub fn fail_fetch(error: anyhow::Error) -> Self { Self { - exit_code: i32::default(), + exit_code: ExitCode::Success, quarantine_status: QuarantineBulkTestStatus::default(), failures: Vec::default(), repo: RepoUrlParts::default(), @@ -300,11 +300,11 @@ impl FailedTestsExtractor { &self.failed_tests } - pub fn exit_code(&self) -> i32 { + pub fn exit_code(&self) -> ExitCode { if self.failed_tests.is_empty() { - EXIT_SUCCESS + ExitCode::Success } else { - EXIT_FAILURE + ExitCode::Failure } } } @@ -314,7 +314,7 @@ pub async fn gather_quarantine_context( request: &api::message::GetQuarantineConfigRequest, file_set_builder: &FileSetBuilder, failed_tests_extractor: Option, - test_run_exit_code: Option, + test_run_exit_code: Option, variant: &String, hide_test_collection_links: bool, ) -> anyhow::Result { @@ -327,7 +327,7 @@ pub async fn gather_quarantine_context( ) }); - let mut exit_code = test_run_exit_code.unwrap_or(EXIT_SUCCESS); + let mut exit_code = test_run_exit_code.unwrap_or(ExitCode::Success); if file_set_builder.no_files_found() { tracing::info!("No test output files found, not quarantining any tests."); @@ -476,11 +476,11 @@ pub async fn gather_quarantine_context( exit_code ); exit_code - } else if exit_code != EXIT_SUCCESS { + } else if !exit_code.is_success() { tracing::info!( "All test failures were quarantined, overriding exit code to be exit_success" ); - EXIT_SUCCESS + ExitCode::Success } else { exit_code }; @@ -710,7 +710,7 @@ mod tests { use proto::upload_metrics::trunk::QuarantineQueryResult; let success_ctx = QuarantineContext { - exit_code: 0, + exit_code: ExitCode::Success, quarantine_status: QuarantineBulkTestStatus::default(), failures: vec![], repo: RepoUrlParts::default(), @@ -742,7 +742,7 @@ mod tests { ..Default::default() }, fetch_status: QuarantineFetchStatus::FetchSucceeded, - exit_code: 0, + exit_code: ExitCode::Success, failures: vec![], repo: RepoUrlParts::default(), org_url_slug: String::new(), diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index ae4b3dc5..547a194d 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -3,6 +3,8 @@ use http::StatusCode; use schemars::JsonSchema; use serde::Serialize; +use crate::exit_code::ExitCode; + pub(crate) const HELP_TEXT: &str = "For more help, contact us at https://slack.trunk.io/"; pub(crate) const CONNECTION_REFUSED_CONTEXT: &str = concat!("Unable to connect to trunk's server",); @@ -40,7 +42,7 @@ impl InterruptingError { pub struct Context { pub base_message: Option, pub org_url_slug: String, - pub exit_code: Option, + pub exit_code: Option, } pub struct ErrorReport { @@ -121,12 +123,12 @@ impl ErrorKind { } } - pub fn exit_code(self) -> Option { + /// `None` leaves the exit code to the rest of the run: `upload` uses quarantining's, and a run + /// that fails outright exits with `ExitCode::Success`, so that our errors fail open. + pub fn exit_code(self) -> Option { match self { Self::ConnectionRefused | Self::Other => None, - Self::Unauthorized | Self::GitRepository | Self::Interrupted => { - Some(exitcode::SOFTWARE) - } + Self::Unauthorized | Self::GitRepository | Self::Interrupted => Some(ExitCode::Error), } } diff --git a/cli/src/exit_code.rs b/cli/src/exit_code.rs new file mode 100644 index 00000000..ac05ce93 --- /dev/null +++ b/cli/src/exit_code.rs @@ -0,0 +1,225 @@ +//! Every exit code the CLI can return, each documented once. The "Exit codes" section of +//! `--help` and the summary's JSON Schema (`summary-output-schema`) are generated from the doc +//! comments below, so they are written for users. + +use std::{borrow::Cow, fmt}; + +use clap::builder::{StyledStr, styling::Styles}; +use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; +use serde::{Serialize, Serializer}; + +/// Defines `ExitCode` from its variants, keeping each variant's doc comment so `--help` can show +/// it. A variant that carries the code (`Variant(code) => code`) is listed as "Passed through". +macro_rules! exit_codes { + ($( + $(#[doc = $doc:literal])+ + $variant:ident $(($passed_through:ident))? => $code:expr, + )+) => { + /// Two exit codes are equal when the process exits with the same number, so + /// `PassedThrough(1) == Failure`. + #[derive(Debug, Clone, Copy)] + pub enum ExitCode { + $( + $(#[doc = $doc])+ + $variant $((exit_code!(@type $passed_through)))?, + )+ + } + + impl ExitCode { + pub const fn code(self) -> i32 { + match self { + $(Self::$variant $(($passed_through))? => $code,)+ + } + } + + /// Each code's label and the lines of its doc comment. + const DOCS: &[(&str, &[&str])] = &[ + $((exit_code!(@label $code $(, $passed_through)?), &[$($doc),+]),)+ + ]; + } + }; +} + +macro_rules! exit_code { + (@type $passed_through:ident) => { + i32 + }; + (@label $code:expr) => { + stringify!($code) + }; + (@label $code:expr, $passed_through:ident) => { + "Passed through" + }; +} + +exit_codes! { + /// Tests passed, every failure was quarantined, or validate found no errors. upload also + /// exits 0 when tests failed if quarantining is turned off (by --disable-quarantining or + /// for the repo in Trunk) or no test reports were found. Errors connecting to Trunk, and + /// errors the CLI doesn't recognize, also exit 0 so that they don't fail your CI job. + Success => 0, + /// Tests failed and weren't quarantined, and upload wasn't given --test-process-exit-code. + /// Also returned when validate found errors, or when test's command ended without an exit + /// code (e.g. it was killed by a signal). + Failure => 1, + /// test exits with its command's exit code, and upload with --test-process-exit-code's + /// value, unless every failure was quarantined. test does this even when uploading the + /// results fails. + PassedThrough(code) => code, + /// Trunk rejected the token or organization slug, the repo root isn't a git repository, or + /// the run stopped early (e.g. no test reports were found with + /// --allow-empty-test-results=false). + Error => 70, + // 130 and 143 follow the shell convention for a process killed by a signal: 128 plus the + // signal's number, which is 2 for SIGINT and 15 for SIGTERM. + /// The CLI was interrupted (SIGINT, or Ctrl-C on Windows). + Interrupted => 130, + /// The CLI was terminated (SIGTERM), e.g. because the CI job was canceled. + Terminated => 143, + /// The command line was invalid, e.g. an unknown or missing argument. + Usage => 2, + /// The CLI crashed. Please report this to us. + Panic => 101, +} + +impl ExitCode { + pub fn is_success(self) -> bool { + self.code() == Self::Success.code() + } + + /// The "Exit codes" section of `--help`, laid out like clap's own long help. + pub fn help() -> StyledStr { + let styles = Styles::default(); + let header = styles.get_header(); + let literal = styles.get_literal(); + let mut help = format!("{header}Exit codes:{header:#}\n"); + for (label, lines) in Self::DOCS { + help.push_str(&format!(" {literal}{label}{literal:#}\n")); + for line in *lines { + help.push_str(&format!(" {}\n", line.trim())); + } + help.push('\n'); + } + help.truncate(help.trim_end().len()); + help.into() + } + + /// The description of `exitCode` in the summary's JSON Schema, as a Markdown list. + fn schema_description() -> String { + let mut description = String::from("The code the CLI exits with:\n"); + for (label, lines) in Self::DOCS { + let label = if label.parse::().is_ok() { + format!("`{label}`") + } else { + label.to_string() + }; + let text = lines.iter().map(|line| line.trim()).collect::>(); + description.push_str(&format!("\n- {label}: {}", text.join(" "))); + } + description + } +} + +impl PartialEq for ExitCode { + fn eq(&self, other: &Self) -> bool { + self.code() == other.code() + } +} + +impl Eq for ExitCode {} + +impl From for i32 { + fn from(exit_code: ExitCode) -> Self { + exit_code.code() + } +} + +impl fmt::Display for ExitCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.code()) + } +} + +impl Serialize for ExitCode { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_i32(self.code()) + } +} + +impl JsonSchema for ExitCode { + fn schema_name() -> Cow<'static, str> { + "ExitCode".into() + } + + fn json_schema(_: &mut SchemaGenerator) -> Schema { + json_schema!({ + "type": "integer", + "description": Self::schema_description(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codes() { + assert_eq!(ExitCode::Success.code(), 0); + assert_eq!(ExitCode::Failure.code(), 1); + assert_eq!(ExitCode::PassedThrough(3).code(), 3); + assert_eq!(ExitCode::Error.code(), 70); + assert_eq!(ExitCode::Interrupted.code(), 130); + assert_eq!(ExitCode::Terminated.code(), 143); + assert_eq!(ExitCode::Usage.code(), 2); + assert_eq!(ExitCode::Panic.code(), 101); + } + + #[test] + fn exit_codes_are_equal_when_their_numbers_are() { + assert_eq!(ExitCode::PassedThrough(0), ExitCode::Success); + assert_eq!(ExitCode::PassedThrough(1), ExitCode::Failure); + assert_ne!(ExitCode::PassedThrough(2), ExitCode::Failure); + assert!(ExitCode::PassedThrough(0).is_success()); + } + + #[test] + fn serializes_as_its_number() { + assert_eq!(serde_json::to_string(&ExitCode::Error).unwrap(), "70"); + assert_eq!( + serde_json::to_string(&ExitCode::PassedThrough(3)).unwrap(), + "3" + ); + } + + #[test] + fn help_lists_every_code() { + let help = ExitCode::help().to_string(); + for label in ["0", "1", "Passed through", "70", "130", "143", "2", "101"] { + assert!( + help.lines().any(|line| line.trim() == label), + "{label} missing from:\n{help}" + ); + } + } + + #[test] + fn schema_description_lists_every_code() { + let description = ExitCode::schema_description(); + for label in [ + "`0`", + "`1`", + "Passed through", + "`70`", + "`130`", + "`143`", + "`2`", + "`101`", + ] { + assert!( + description.contains(&format!("\n- {label}: ")), + "{label} missing from:\n{description}" + ); + } + } +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index fbc108c6..23f74e5c 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1,6 +1,7 @@ pub mod context; pub mod context_quarantine; pub mod error_report; +pub mod exit_code; pub mod print; pub mod report_limiting; pub mod summary_output; diff --git a/cli/src/main.rs b/cli/src/main.rs index 2817e5d6..b6f99699 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -18,6 +18,7 @@ use tracing_subscriber::{filter::FilterFn, prelude::*}; use trunk_analytics_cli::{ context::gather_debug_props, error_report::ErrorReport, + exit_code::ExitCode, summary_output::{ ErrorSummary, UploadCommand, UploadReport, ValidateReport, json::{self, Report}, @@ -32,6 +33,7 @@ use trunk_analytics_cli::{ #[command( version = std::env!("CARGO_PKG_VERSION"), about = "Trunk Analytics CLI", + after_long_help = ExitCode::help(), )] struct Cli { #[command(subcommand)] @@ -102,17 +104,20 @@ impl Cli { #[derive(Debug, Subcommand)] enum Commands { /// Run a test command and upload results to Trunk Flaky Tests. Automatically detects and reports flaky tests. + #[command(after_long_help = ExitCode::help())] Test(TestArgs), /// Upload test results to Trunk Flaky Tests. Use this when you've already run tests and have result files. + #[command(after_long_help = ExitCode::help())] Upload(UploadArgs), /// Validate test report files for compatibility with Trunk Flaky Tests. Runs locally without uploading data. + #[command(after_long_help = ExitCode::help())] Validate(ValidateArgs), /// Print the JSON Schema of the summary that `--summary-output-file` writes. SummaryOutputSchema, } fn close_out_and_exit( - exit_code: i32, + exit_code: ExitCode, guard: ClientInitGuard, render_sender: Sender, render_handle: JoinHandle<()>, @@ -120,7 +125,7 @@ fn close_out_and_exit( guard.flush(None); drop(render_sender); let _ = render_handle.join(); - std::process::exit(exit_code) + std::process::exit(exit_code.code()) } // "the Sentry client must be initialized before starting an async runtime or spawning threads" @@ -218,7 +223,7 @@ fn main() -> anyhow::Result<()> { } MainOutcome::Finished(Ok(RunResult::SummaryOutputSchema)) => { println!("{}", serde_json::to_string_pretty(&json::schema())?); - close_out_and_exit(exitcode::OK, guard, render_sender, render_handle) + close_out_and_exit(ExitCode::Success, guard, render_sender, render_handle) } MainOutcome::Finished(Err(error)) => { let error = Arc::new(ErrorSummary::new(&ErrorReport::new( @@ -226,7 +231,7 @@ fn main() -> anyhow::Result<()> { org_url_slug, None, ))); - let exit_code = error.exit_code.unwrap_or(exitcode::OK); + let exit_code = error.exit_code.unwrap_or(ExitCode::Success); send_message( DisplayMessage::Final(error.clone(), String::from("error display")), &render_sender, @@ -258,7 +263,7 @@ enum MainOutcome { /// The command ran to completion (successfully or with an error). Finished(anyhow::Result), /// A termination signal was received first; the payload is the exit code to use. - Interrupted(i32), + Interrupted(ExitCode), } enum RunResult { diff --git a/cli/src/shutdown.rs b/cli/src/shutdown.rs index 0ae81924..2bfb8552 100644 --- a/cli/src/shutdown.rs +++ b/cli/src/shutdown.rs @@ -4,16 +4,11 @@ //! these signals against the actual work in `main` so a cancellation aborts in-flight //! operations (e.g. the API retry loop) immediately instead of blocking until they finish. -// Conventional process exit codes for termination by signal: `128 + signal number`. This -// POSIX/shell convention isn't part of sysexits.h, so the `exitcode` crate used elsewhere -// doesn't define it. SIGINT is 2 and SIGTERM is 15 on the Unix platforms we target. -const EXIT_SIGNAL_BASE: i32 = 128; -const EXIT_CODE_SIGINT: i32 = EXIT_SIGNAL_BASE + 2; -const EXIT_CODE_SIGTERM: i32 = EXIT_SIGNAL_BASE + 15; +use trunk_analytics_cli::exit_code::ExitCode; /// Resolves once the process receives a termination signal β€” SIGINT/SIGTERM on Unix, Ctrl-C /// on Windows. Returns the conventional `128 + signal` exit code. -pub async fn shutdown_signal() -> i32 { +pub async fn shutdown_signal() -> ExitCode { #[cfg(unix)] { use tokio::signal::unix::{SignalKind, signal}; @@ -36,14 +31,14 @@ pub async fn shutdown_signal() -> i32 { } }; tokio::select! { - _ = sigterm.recv() => EXIT_CODE_SIGTERM, - _ = sigint.recv() => EXIT_CODE_SIGINT, + _ = sigterm.recv() => ExitCode::Terminated, + _ = sigint.recv() => ExitCode::Interrupted, } } #[cfg(not(unix))] { // Ctrl-C on Windows is the SIGINT-equivalent interruption. let _ = tokio::signal::ctrl_c().await; - EXIT_CODE_SIGINT + ExitCode::Interrupted } } diff --git a/cli/src/summary_output/error.rs b/cli/src/summary_output/error.rs index 22790f88..0985de3b 100644 --- a/cli/src/summary_output/error.rs +++ b/cli/src/summary_output/error.rs @@ -2,7 +2,10 @@ use api::client::get_api_host; use schemars::JsonSchema; use serde::Serialize; -use crate::error_report::{ErrorKind, ErrorReport, get_client_error_explanation, settings_url}; +use crate::{ + error_report::{ErrorKind, ErrorReport, get_client_error_explanation, settings_url}, + exit_code::ExitCode, +}; #[derive(Debug, Clone, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] @@ -19,7 +22,7 @@ pub struct ErrorSummary { /// Where to check the org slug and token. Only set for `UNAUTHORIZED` errors. pub settings_url: Option, /// A run that fails outright exits with this code, or 0 when it's null. Upload reports exit with their own `exitCode`. - pub exit_code: Option, + pub exit_code: Option, } impl ErrorSummary { diff --git a/cli/src/summary_output/text.rs b/cli/src/summary_output/text.rs index f97c2dea..311ed3a2 100644 --- a/cli/src/summary_output/text.rs +++ b/cli/src/summary_output/text.rs @@ -1,6 +1,5 @@ use std::path::{Path, PathBuf}; -use constants::EXIT_SUCCESS; use display::end_output::EndOutput; use pluralizer::pluralize; use superconsole::{ @@ -14,6 +13,7 @@ use super::{ }; use crate::{ error_report::{CONNECTION_REFUSED_CONTEXT, ErrorKind, HELP_TEXT, UNAUTHORIZED_CONTEXT}, + exit_code::ExitCode, report_limiting::ValidationReport, upload_command::{UploadRunResult, get_bundle_upload_id_message}, validate_command::ValidateRunResult, @@ -141,9 +141,9 @@ pub fn render_error(error: &ErrorSummary) -> anyhow::Result> { lines.push(Line::from_iter([Span::new_unstyled(HELP_TEXT)?])); lines.push(Line::default()); match error.exit_code { - Some(exitcode::OK) => lines.push(Line::from_iter([ + Some(exit_code) if exit_code.is_success() => lines.push(Line::from_iter([ Span::new_unstyled("No errors occurred, returning default exit code: ")?, - Span::new_styled(style(exitcode::OK.to_string()).attribute(Attribute::Bold))?, + Span::new_styled(style(exit_code.to_string()).attribute(Attribute::Bold))?, ])), Some(exit_code) => lines.push(Line::from_iter([ Span::new_unstyled("Errors occurred during execution, returning exit code: ")?, @@ -277,7 +277,7 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re Span::new_unstyled( "πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success ", )?, - Span::new_styled(style(format!("({})", EXIT_SUCCESS)).attribute(Attribute::Bold))?, + Span::new_styled(style(format!("({})", ExitCode::Success)).attribute(Attribute::Bold))?, ])), Outcome::NoFailures => output.push(Line::from_iter([Span::new_unstyled( "πŸŽ‰ No test failures found!", diff --git a/cli/src/summary_output/upload.rs b/cli/src/summary_output/upload.rs index 58e55015..9810b216 100644 --- a/cli/src/summary_output/upload.rs +++ b/cli/src/summary_output/upload.rs @@ -2,13 +2,14 @@ use std::collections::BTreeMap; use api::urls::{url_for_test_case, url_for_upload}; use bundle::Test; -use constants::{EXIT_FAILURE, EXIT_SUCCESS}; use context::junit::validator::JunitValidationLevel; use schemars::JsonSchema; use serde::Serialize; use super::{ErrorSummary, Notice}; -use crate::{upload_command::UploadRunResult, validate_command::JunitReportValidations}; +use crate::{ + exit_code::ExitCode, upload_command::UploadRunResult, validate_command::JunitReportValidations, +}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] @@ -43,7 +44,7 @@ pub struct UploadReport { pub outcome: Outcome, /// The code the CLI exits with. Left out by the RSpec gem, where RSpec decides the exit code. #[serde(skip_serializing_if = "Option::is_none")] - pub exit_code: Option, + pub exit_code: Option, /// Warnings that didn't stop the run, such as a deprecated flag. pub notices: Vec, } @@ -199,8 +200,8 @@ impl UploadReport { impl UploadReport { /// What `upload` and `test` exit with; only the RSpec gem leaves `exit_code` unset. - pub fn cli_exit_code(&self) -> i32 { - self.exit_code.unwrap_or(EXIT_FAILURE) + pub fn cli_exit_code(&self) -> ExitCode { + self.exit_code.unwrap_or(ExitCode::Failure) } } @@ -243,12 +244,12 @@ impl ParseError { } impl Outcome { - fn new(counts: &TestCounts, test_run_exit_code: i32) -> Self { + fn new(counts: &TestCounts, test_run_exit_code: ExitCode) -> Self { if counts.total == 0 { Self::NoTestsFound } else if counts.failed == 0 && counts.quarantined > 0 { Self::AllFailuresQuarantined - } else if counts.failed == 0 && test_run_exit_code == EXIT_SUCCESS { + } else if counts.failed == 0 && test_run_exit_code.is_success() { Self::NoFailures } else if counts.failed == 0 { Self::NoFailuresButNonZeroExitCode diff --git a/cli/src/summary_output/validate.rs b/cli/src/summary_output/validate.rs index 171c2a00..e2d732c7 100644 --- a/cli/src/summary_output/validate.rs +++ b/cli/src/summary_output/validate.rs @@ -1,11 +1,13 @@ use std::collections::HashSet; -use constants::{EXIT_FAILURE, EXIT_SUCCESS}; use context::junit::validator::JunitReportValidationFlatIssue; use serde::Serialize; use super::{FileValidation, Notice, ParseError}; -use crate::validate_command::{ParseIssues, TestIssues, ValidateRunResult}; +use crate::{ + exit_code::ExitCode, + validate_command::{ParseIssues, TestIssues, ValidateRunResult}, +}; #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -14,7 +16,7 @@ pub struct ValidateReport { pub validations: Vec, pub codeowners: Option, pub counts: ValidationCounts, - pub exit_code: i32, + pub exit_code: ExitCode, pub notices: Vec, } @@ -80,9 +82,9 @@ impl ValidateReport { warnings: codeowners.warnings.clone(), }), exit_code: if has_errors { - EXIT_FAILURE + ExitCode::Failure } else { - EXIT_SUCCESS + ExitCode::Success }, notices, } diff --git a/cli/src/test_command.rs b/cli/src/test_command.rs index 98c2c071..716b59e9 100644 --- a/cli/src/test_command.rs +++ b/cli/src/test_command.rs @@ -7,7 +7,6 @@ use std::{ }; use clap::Args; -use constants::EXIT_FAILURE; use display::{ end_output::EndOutput, message::{DisplayMessage, send_message}, @@ -19,6 +18,7 @@ use superconsole::{ use crate::{ context::{gather_debug_props, gather_initial_test_context}, + exit_code::ExitCode, upload_command::{RunUploadOptions, UploadArgs, UploadRunResult, run_upload}, }; @@ -186,14 +186,14 @@ pub async fn run_test_command>( let (exit_code, stdout, stderr) = exit_result .map(|result| { ( - result.status.code().unwrap_or(EXIT_FAILURE), + result.status.code().unwrap_or(ExitCode::Failure.code()), String::from_utf8(result.stdout) .unwrap_or_else(|_| String::from("Error: Command had non-utf-8 output!")), String::from_utf8(result.stderr) .unwrap_or_else(|_| String::from("Error: Command had non-utf-8 error output!")), ) }) - .unwrap_or((EXIT_FAILURE, String::from(""), String::from(""))); + .unwrap_or((ExitCode::Failure.code(), String::from(""), String::from(""))); tracing::info!("Command exit code: {}", exit_code); Ok(TestRunResult { diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index 0f85ebaa..c2a81b62 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -9,13 +9,13 @@ use api::urls::TestCaseGuidScope; use bundle::{BundleMeta, BundlerUtil, QuarantineResolutionMode, unzip_tarball}; use clap::{ArgAction, Args}; use codeowners::OwnersSource; -use constants::EXIT_SUCCESS; use context::bazel_bep::common::BepParseResult; use context::env::parser::CIPlatform; use display::message::DisplayMessage; use tempfile::TempDir; use crate::context_quarantine::{QuarantineContext, quarantine_query_result}; +use crate::exit_code::ExitCode; use crate::validate_command::JunitReportValidations; use crate::{ context::{ @@ -640,7 +640,8 @@ pub async fn run_upload( let default_exit_code = upload_args .test_process_exit_code - .or_else(|| test_run_result.as_ref().map(|r| r.exit_code)); + .or_else(|| test_run_result.as_ref().map(|r| r.exit_code)) + .map(ExitCode::PassedThrough); let disable_quarantining = upload_args.disable_quarantining || !upload_args.use_quarantining.unwrap_or(true); let quarantine_context = match gather_exit_code_and_quarantined_tests_context( @@ -888,7 +889,7 @@ async fn upload_bundle( requested_test_collection_short_id: Option, api_client: &ApiClient, bep_result: Option, - exit_code: i32, + exit_code: ExitCode, dry_run: bool, ) -> anyhow::Result { let upload_result = gather_upload_id_context( @@ -920,7 +921,7 @@ async fn upload_bundle( api_client .put_bundle_to_s3(&upload.url, &bundle_temp_file) .await?; - if exit_code == EXIT_SUCCESS { + if exit_code.is_success() { tracing::info!("Upload successful"); } else { tracing::info!( diff --git a/cli/tests/summary/error_report.rs b/cli/tests/summary/error_report.rs index 7e155f7e..8340e6a4 100644 --- a/cli/tests/summary/error_report.rs +++ b/cli/tests/summary/error_report.rs @@ -3,6 +3,7 @@ use pretty_assertions::assert_eq; use superconsole::Line; use trunk_analytics_cli::{ error_report::{ErrorReport, InterruptingError}, + exit_code::ExitCode, summary_output::{ ErrorSummary, json::{self, Report}, @@ -152,7 +153,7 @@ fn prints_the_default_exit_code() { String::from("fake-org-slug"), None, ); - report.context.exit_code = Some(exitcode::OK); + report.context.exit_code = Some(ExitCode::Success); assert_eq!( error_summary(&report), @@ -173,7 +174,7 @@ fn prints_any_other_exit_code() { String::from("fake-org-slug"), None, ); - report.context.exit_code = Some(exitcode::USAGE); + report.context.exit_code = Some(ExitCode::PassedThrough(64)); assert_eq!( error_summary(&report), diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index 4e3e1032..03c11447 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -18,6 +18,7 @@ use superconsole::Line; use trunk_analytics_cli::{ context_quarantine::{QuarantineContext, QuarantineFetchStatus}, error_report::ErrorReport, + exit_code::ExitCode, report_limiting::ValidationReport, summary_output::{ Notice, UploadCommand, UploadReport, @@ -283,7 +284,7 @@ fn prints_failures_that_were_all_quarantined() { #[test] fn prints_failures_that_were_not_quarantined() { let mut result = upload_result(5); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; result .quarantine_context @@ -326,7 +327,7 @@ fn prints_failures_that_were_not_quarantined() { #[test] fn prints_a_warning_when_the_quarantine_lookup_failed() { let mut result = upload_result(3); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchFailed(anyhow::anyhow!("request timed out")); result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; @@ -355,7 +356,7 @@ fn prints_a_warning_when_the_quarantine_lookup_failed() { #[test] fn stops_at_an_error_that_blocks_quarantining() { let mut result = upload_result(3); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; result.error_report = Some(ErrorReport::new( unauthorized_error(), @@ -380,7 +381,7 @@ fn stops_at_an_error_that_blocks_quarantining() { fn proceeds_past_an_upload_error() { let mut result = upload_result(3); result.meta.base_props.bundle_upload_id = String::new(); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; result.error_report = Some(ErrorReport::new( anyhow::anyhow!("HTTP status client error (400 Bad Request)") @@ -557,7 +558,7 @@ fn prints_validation_when_every_file_is_correct() { fn prints_long_failure_messages_when_asked() { let mut result = upload_result(1); result.show_failure_messages = true; - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; let mut message_lines = vec![ String::from("assertion `left == right` failed\r"), String::from("\tleft: 1"), @@ -614,7 +615,7 @@ fn prints_long_failure_messages_when_asked() { #[test] fn prints_the_upload_as_a_collection_link() { let mut result = in_collection(upload_result(2)); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.failures = vec![Test { id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), ..failure("adds", "MathTests", None) @@ -647,7 +648,7 @@ fn prints_the_upload_as_a_collection_link() { #[test] fn links_tests_by_short_link_without_collection_ids() { let mut result = in_collection(upload_result(2)); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; assert_eq!( @@ -674,7 +675,7 @@ fn links_tests_by_short_link_without_collection_ids() { fn prints_the_bundle_upload_id_when_collection_links_are_hidden() { let mut result = in_collection(upload_result(2)); result.hide_test_collection_links = true; - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.failures = vec![Test { id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), ..failure("adds", "MathTests", None) @@ -723,7 +724,7 @@ fn omits_the_upload_line_without_a_bundle_upload_id() { #[test] fn groups_failures_by_file_then_suite_then_neither() { let mut result = upload_result(10); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.failures = vec![ failure("parses_dates", "DateTests", Some("src/date.rs")), failure("adds", "MathTests", None), @@ -775,7 +776,7 @@ fn groups_failures_by_file_then_suite_then_neither() { #[test] fn shows_three_failures_per_group() { let mut result = upload_result(5); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.failures = ["adds", "subtracts", "multiplies", "divides", "rounds"] .into_iter() .map(|name| failure(name, "MathTests", Some("src/math.rs"))) @@ -809,7 +810,7 @@ fn shows_three_failures_per_group() { #[test] fn warns_about_a_non_zero_exit_code_without_failures() { let mut result = upload_result(5); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; assert_eq!( upload_summary(&result), @@ -828,7 +829,7 @@ fn warns_about_a_non_zero_exit_code_without_failures() { #[test] fn upload_exits_with_the_exit_code_of_an_error_but_test_does_not() { let mut result = upload_result(3); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.error_report = Some(ErrorReport::new( unauthorized_error(), String::from("test-org"), @@ -837,18 +838,18 @@ fn upload_exits_with_the_exit_code_of_an_error_but_test_does_not() { assert_eq!( UploadReport::new(&result, UploadCommand::Upload).exit_code, - Some(70) + Some(ExitCode::Error) ); assert_eq!( UploadReport::new(&result, UploadCommand::Test).exit_code, - Some(1) + Some(ExitCode::Failure) ); } #[test] fn exits_with_the_test_run_exit_code_when_an_error_has_none() { let mut result = upload_result(3); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.error_report = Some(ErrorReport::new( anyhow::anyhow!("HTTP status server error (500 Internal Server Error)"), String::from("test-org"), @@ -857,11 +858,11 @@ fn exits_with_the_test_run_exit_code_when_an_error_has_none() { assert_eq!( UploadReport::new(&result, UploadCommand::Upload).exit_code, - Some(1) + Some(ExitCode::Failure) ); assert_eq!( UploadReport::new(&result, UploadCommand::Test).exit_code, - Some(1) + Some(ExitCode::Failure) ); } @@ -892,7 +893,7 @@ fn writes_the_report_as_json() { Err(anyhow::anyhow!("unexpected end of file")), ), ])); - result.quarantine_context.exit_code = 1; + result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; result .quarantine_context diff --git a/cli/tests/upload.rs b/cli/tests/upload.rs index 14f59427..2bfd0a99 100644 --- a/cli/tests/upload.rs +++ b/cli/tests/upload.rs @@ -38,7 +38,6 @@ const SINGLE_TEST_JUNIT_XML: &str = r#" "#; -use constants::EXIT_FAILURE; use context::{ bazel_bep::{common::BepTestStatus, parser::BazelBepParser}, junit::{junit_path::TestRunnerReportStatus, parser::JunitParser}, @@ -60,6 +59,7 @@ use test_utils::{ SharedMockServerState, }, }; +use trunk_analytics_cli::exit_code::ExitCode; use trunk_analytics_cli::upload_command::{DRY_RUN_OUTPUT_DIR, get_bundle_upload_id_message}; // The mock JUnit generator adds a random number of reruns per test, so the number of @@ -1647,13 +1647,13 @@ async fn upload_writes_an_error_to_summary_output_file() { .arg("--summary-output-file") .arg(&summary_path) .assert() - .code(exitcode::SOFTWARE); + .code(ExitCode::Error.code()); let summary = read_summary_output_file(&summary_path); assert_eq!(summary["schemaVersion"], 1); assert_eq!(summary["type"], "ERROR"); assert_eq!(summary["kind"], "GIT_REPOSITORY"); - assert_eq!(summary["exitCode"], exitcode::SOFTWARE); + assert_eq!(summary["exitCode"], ExitCode::Error.code()); } #[tokio::test(flavor = "multi_thread")] @@ -2809,7 +2809,7 @@ async fn uses_failure_exit_code_if_unquarantined_tests_fail() { .junit_paths("junit.xml") .command() .assert() - .code(predicate::eq(EXIT_FAILURE)); + .code(predicate::eq(ExitCode::Failure.code())); // HINT: View CLI output with `cargo test -- --nocapture` println!("{assert}"); diff --git a/constants/src/lib.rs b/constants/src/lib.rs index b726fc70..9ada3b06 100644 --- a/constants/src/lib.rs +++ b/constants/src/lib.rs @@ -1,8 +1,5 @@ pub const ALLOW_LIST: &[&str] = &[r".*\.xml$", r".*\.junit$", r".*\.bin$"]; -pub const EXIT_SUCCESS: i32 = 0; -pub const EXIT_FAILURE: i32 = 1; - // NOTE: The order of these locations matters! Each provider looks at each of these locations in // order, to find the relevant `CODEOWNERS` file. // From 3dea1346d8537c8df5738b0b2af69c66d4eb033d Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Sat, 24 Oct 2026 00:34:17 +0000 Subject: [PATCH 16/19] refactor(cli): read the exit code docs with strum instead of a macro strum's EnumMessage derive exposes each variant's doc comment through get_documentation(), and EnumIter lists the variants, so ExitCode no longer needs macro_rules! to keep its doc comments as strings. The numbers move into a plain match in code(). --help and summary-output-schema print exactly the same text as before. strum 0.27.2 was already in Cargo.lock through other dependencies. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + cli/Cargo.toml | 1 + cli/src/exit_code.rs | 109 +++++++++++++++++++------------------------ 3 files changed, 49 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7469eef7..bdcedeeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5973,6 +5973,7 @@ dependencies = [ "sentry-tracing", "serde", "serde_json", + "strum 0.27.2", "superconsole", "tempfile", "test_utils", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index e3d30154..61476822 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -52,6 +52,7 @@ quick-junit = "0.5.0" schemars = "1.2" serde = { version = "1.0.215", features = ["derive"] } serde_json = "1.0" +strum = { version = "0.27", features = ["derive"] } clap-verbosity-flag = "3.0.2" proto = { path = "../proto" } regex = "1.11.1" diff --git a/cli/src/exit_code.rs b/cli/src/exit_code.rs index ac05ce93..96cbca11 100644 --- a/cli/src/exit_code.rs +++ b/cli/src/exit_code.rs @@ -7,82 +7,55 @@ use std::{borrow::Cow, fmt}; use clap::builder::{StyledStr, styling::Styles}; use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Serialize, Serializer}; +use strum::{EnumIter, EnumMessage, IntoEnumIterator}; -/// Defines `ExitCode` from its variants, keeping each variant's doc comment so `--help` can show -/// it. A variant that carries the code (`Variant(code) => code`) is listed as "Passed through". -macro_rules! exit_codes { - ($( - $(#[doc = $doc:literal])+ - $variant:ident $(($passed_through:ident))? => $code:expr, - )+) => { - /// Two exit codes are equal when the process exits with the same number, so - /// `PassedThrough(1) == Failure`. - #[derive(Debug, Clone, Copy)] - pub enum ExitCode { - $( - $(#[doc = $doc])+ - $variant $((exit_code!(@type $passed_through)))?, - )+ - } - - impl ExitCode { - pub const fn code(self) -> i32 { - match self { - $(Self::$variant $(($passed_through))? => $code,)+ - } - } - - /// Each code's label and the lines of its doc comment. - const DOCS: &[(&str, &[&str])] = &[ - $((exit_code!(@label $code $(, $passed_through)?), &[$($doc),+]),)+ - ]; - } - }; -} - -macro_rules! exit_code { - (@type $passed_through:ident) => { - i32 - }; - (@label $code:expr) => { - stringify!($code) - }; - (@label $code:expr, $passed_through:ident) => { - "Passed through" - }; -} - -exit_codes! { +/// Two exit codes are equal when the process exits with the same number, so +/// `PassedThrough(1) == Failure`. +#[derive(Debug, Clone, Copy, EnumIter, EnumMessage)] +pub enum ExitCode { /// Tests passed, every failure was quarantined, or validate found no errors. upload also /// exits 0 when tests failed if quarantining is turned off (by --disable-quarantining or /// for the repo in Trunk) or no test reports were found. Errors connecting to Trunk, and /// errors the CLI doesn't recognize, also exit 0 so that they don't fail your CI job. - Success => 0, + Success, /// Tests failed and weren't quarantined, and upload wasn't given --test-process-exit-code. /// Also returned when validate found errors, or when test's command ended without an exit /// code (e.g. it was killed by a signal). - Failure => 1, + Failure, /// test exits with its command's exit code, and upload with --test-process-exit-code's /// value, unless every failure was quarantined. test does this even when uploading the /// results fails. - PassedThrough(code) => code, + PassedThrough(i32), /// Trunk rejected the token or organization slug, the repo root isn't a git repository, or /// the run stopped early (e.g. no test reports were found with /// --allow-empty-test-results=false). - Error => 70, - // 130 and 143 follow the shell convention for a process killed by a signal: 128 plus the - // signal's number, which is 2 for SIGINT and 15 for SIGTERM. + Error, /// The CLI was interrupted (SIGINT, or Ctrl-C on Windows). - Interrupted => 130, + Interrupted, /// The CLI was terminated (SIGTERM), e.g. because the CI job was canceled. - Terminated => 143, + Terminated, /// The command line was invalid, e.g. an unknown or missing argument. - Usage => 2, + Usage, /// The CLI crashed. Please report this to us. - Panic => 101, + Panic, } impl ExitCode { + pub const fn code(self) -> i32 { + match self { + Self::Success => 0, + Self::Failure => 1, + Self::PassedThrough(code) => code, + Self::Error => 70, + // 130 and 143 follow the shell convention for a process killed by a signal: 128 + // plus the signal's number, which is 2 for SIGINT and 15 for SIGTERM. + Self::Interrupted => 130, + Self::Terminated => 143, + Self::Usage => 2, + Self::Panic => 101, + } + } + pub fn is_success(self) -> bool { self.code() == Self::Success.code() } @@ -93,10 +66,10 @@ impl ExitCode { let header = styles.get_header(); let literal = styles.get_literal(); let mut help = format!("{header}Exit codes:{header:#}\n"); - for (label, lines) in Self::DOCS { + for (label, lines) in Self::docs() { help.push_str(&format!(" {literal}{label}{literal:#}\n")); - for line in *lines { - help.push_str(&format!(" {}\n", line.trim())); + for line in lines { + help.push_str(&format!(" {line}\n")); } help.push('\n'); } @@ -107,17 +80,29 @@ impl ExitCode { /// The description of `exitCode` in the summary's JSON Schema, as a Markdown list. fn schema_description() -> String { let mut description = String::from("The code the CLI exits with:\n"); - for (label, lines) in Self::DOCS { + for (label, lines) in Self::docs() { let label = if label.parse::().is_ok() { format!("`{label}`") } else { - label.to_string() + label }; - let text = lines.iter().map(|line| line.trim()).collect::>(); - description.push_str(&format!("\n- {label}: {}", text.join(" "))); + let text = lines.collect::>().join(" "); + description.push_str(&format!("\n- {label}: {text}")); } description } + + /// Each code's label and the lines of its doc comment. + fn docs() -> impl Iterator)> { + Self::iter().map(|exit_code| { + let label = match exit_code { + Self::PassedThrough(_) => String::from("Passed through"), + _ => exit_code.code().to_string(), + }; + let lines = exit_code.get_documentation().unwrap_or_default().lines(); + (label, lines.map(str::trim)) + }) + } } impl PartialEq for ExitCode { From 46db1082692e1d13a2f108ef73cd0468a2e2270e Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Fri, 23 Oct 2026 22:55:30 +0000 Subject: [PATCH 17/19] fix(cli): say when no tests were found but the exit code isn't 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the test reports have no tests, the summary only said "No tests were found in the provided test results", even when the run exits with the test step's non-zero code. A test step that failed before running any tests, such as a Playwright global setup that timed out, looked like an empty report rather than a failure. The summary now names the exit code and points at the step's own output: ⚠️ No tests were found in the provided test results, but non zero exit code provided: 1 The test step appears to have failed without running tests -- see above for the original failure. With exit code 0, the single line is unchanged. The code is the one the CLI exits with, which is the test run's code wherever the outcome is printed: an error with its own exit code stops the summary before the outcome. The JSON summary already has `outcome: NO_TESTS_FOUND` and `exitCode`, so it doesn't change. Refs TRUNK-19074 Co-Authored-By: Claude Opus 5.5 (1M context) --- cli/src/summary_output/text.rs | 21 ++++++++++++++++++--- cli/tests/summary/upload.rs | 19 +++++++++++++++++++ cli/tests/test.rs | 5 ++++- cli/tests/upload.rs | 3 ++- 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/cli/src/summary_output/text.rs b/cli/src/summary_output/text.rs index 311ed3a2..08d7596b 100644 --- a/cli/src/summary_output/text.rs +++ b/cli/src/summary_output/text.rs @@ -270,9 +270,24 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re } match report.outcome { - Outcome::NoTestsFound => output.push(Line::from_iter([Span::new_unstyled( - "⚠️ No tests were found in the provided test results", - )?])), + Outcome::NoTestsFound if report.cli_exit_code().is_success() => { + output.push(Line::from_iter([Span::new_unstyled( + "⚠️ No tests were found in the provided test results", + )?])) + } + Outcome::NoTestsFound => { + output.push(Line::from_iter([ + Span::new_unstyled( + "⚠️ No tests were found in the provided test results, but non zero exit code provided: ", + )?, + Span::new_styled( + style(format!("{}", report.cli_exit_code())).attribute(Attribute::Bold), + )?, + ])); + output.push(Line::from_iter([Span::new_unstyled( + "The test step appears to have failed without running tests -- see above for the original failure.", + )?])); + } Outcome::AllFailuresQuarantined => output.push(Line::from_iter([ Span::new_unstyled( "πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success ", diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index 03c11447..a522cdf8 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -243,6 +243,25 @@ fn prints_that_no_tests_were_found() { ); } +#[test] +fn prints_that_no_tests_were_found_with_a_non_zero_exit_code() { + let mut result = upload_result(0); + result.quarantine_context.exit_code = ExitCode::Failure; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 0 Pass: 0 Fail: 0 Quarantined: 0 Pass Ratio: N/A", + "", + "⚠️ No tests were found in the provided test results, but non zero exit code provided: 1", + "The test step appears to have failed without running tests -- see above for the original failure.", + ] + ); +} + #[test] fn prints_failures_that_were_all_quarantined() { let mut result = upload_result(4); diff --git a/cli/tests/test.rs b/cli/tests/test.rs index c8277514..4286b472 100644 --- a/cli/tests/test.rs +++ b/cli/tests/test.rs @@ -153,7 +153,10 @@ async fn test_command_fails_with_no_junit_files_no_quarantine_successful_upload( .failure() .code(128) .stderr(predicate::str::contains( - "No tests were found in the provided test results", + "No tests were found in the provided test results, but non zero exit code provided: ", + )) + .stderr(predicate::str::contains( + "The test step appears to have failed without running tests", )); println!("{assert}"); diff --git a/cli/tests/upload.rs b/cli/tests/upload.rs index 2bfd0a99..b2d8ca91 100644 --- a/cli/tests/upload.rs +++ b/cli/tests/upload.rs @@ -1729,7 +1729,8 @@ async fn upload_bundle_with_no_junit_files_no_quarantine_successful_upload() { .success() .stderr(predicate::str::contains( "No tests were found in the provided test results", - )); + )) + .stderr(predicate::str::contains("without running tests").not()); // HINT: View CLI output with `cargo test -- --nocapture` println!("{assert}"); From 3a907603bf1481fc56223f88aff620f1930e5056 Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Tue, 27 Oct 2026 21:17:23 +0000 Subject: [PATCH 18/19] feat(cli): add a hidden github-annotations command `trunk-analytics-cli github-annotations --summary-file ` reads the summary that --summary-output-file wrote and prints GitHub Actions workflow commands saying why the run failed. The analytics-uploader action calls it when the CLI exits non-zero, so the CLI words the failure and the action never reads the summary's format. - Exits 0 only when it printed an annotation. A summary with nothing to say, or one it can't read, exits 1 so the action falls back to its own message. - Runs before Sentry, logging and the renderer start, so stdout carries only the workflow commands. - Follows the text summary: an error that blocks quarantining is the only annotation; any other upload error is a warning; then the outcome. Each failure that wasn't quarantined gets its own annotation with its file, up to 9, since GitHub shows 10 errors per step. - The wording is shared with the text summary through constants in report/text.rs, including the "failed without running tests" line. To read the file back, the report types derive Deserialize, ExitCode reads a number back as its named code (or PassedThrough), and json::from_str rejects a schemaVersion newer than it knows. Refs TRUNK-19074 Co-Authored-By: Claude Opus 5.5 --- cli/src/error_report.rs | 4 +- cli/src/exit_code.rs | 22 ++- cli/src/main.rs | 42 ++++- cli/src/summary_output/error.rs | 4 +- cli/src/summary_output/github.rs | 229 +++++++++++++++++++++++++++ cli/src/summary_output/json.rs | 36 ++++- cli/src/summary_output/mod.rs | 1 + cli/src/summary_output/notice.rs | 6 +- cli/src/summary_output/text.rs | 41 +++-- cli/src/summary_output/upload.rs | 22 +-- cli/tests/summary/github.rs | 257 +++++++++++++++++++++++++++++++ cli/tests/summary/main.rs | 1 + cli/tests/summary/upload.rs | 6 +- 13 files changed, 625 insertions(+), 46 deletions(-) create mode 100644 cli/src/summary_output/github.rs create mode 100644 cli/tests/summary/github.rs diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index 547a194d..130540b5 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -1,7 +1,7 @@ use api::client::ClientErrorExplanation; use http::StatusCode; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::exit_code::ExitCode; @@ -94,7 +94,7 @@ impl ErrorReport { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum ErrorKind { /// Couldn't connect to Trunk. diff --git a/cli/src/exit_code.rs b/cli/src/exit_code.rs index 96cbca11..a8c5f9ec 100644 --- a/cli/src/exit_code.rs +++ b/cli/src/exit_code.rs @@ -6,7 +6,7 @@ use std::{borrow::Cow, fmt}; use clap::builder::{StyledStr, styling::Styles}; use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; -use serde::{Serialize, Serializer}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use strum::{EnumIter, EnumMessage, IntoEnumIterator}; /// Two exit codes are equal when the process exits with the same number, so @@ -131,6 +131,18 @@ impl Serialize for ExitCode { } } +/// A number that matches a named code reads back as that code, and any other as `PassedThrough`. +impl<'de> Deserialize<'de> for ExitCode { + fn deserialize>(deserializer: D) -> Result { + let code = i32::deserialize(deserializer)?; + Ok(Self::iter() + .find(|exit_code| { + !matches!(exit_code, Self::PassedThrough(_)) && exit_code.code() == code + }) + .unwrap_or(Self::PassedThrough(code))) + } +} + impl JsonSchema for ExitCode { fn schema_name() -> Cow<'static, str> { "ExitCode".into() @@ -168,6 +180,14 @@ mod tests { assert!(ExitCode::PassedThrough(0).is_success()); } + #[test] + fn deserializes_from_its_number() { + let read = |json: &str| serde_json::from_str::(json).unwrap(); + assert!(matches!(read("70"), ExitCode::Error)); + assert!(matches!(read("0"), ExitCode::Success)); + assert!(matches!(read("3"), ExitCode::PassedThrough(3))); + } + #[test] fn serializes_as_its_number() { assert_eq!(serde_json::to_string(&ExitCode::Error).unwrap(), "70"); diff --git a/cli/src/main.rs b/cli/src/main.rs index b6f99699..425ef7dc 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -7,7 +7,7 @@ use std::{ thread::JoinHandle, }; -use clap::{Parser, Subcommand}; +use clap::{Args, Parser, Subcommand}; use clap_verbosity_flag::{InfoLevel, Verbosity, log::LevelFilter}; use display::{ message::{DisplayMessage, send_message}, @@ -20,7 +20,7 @@ use trunk_analytics_cli::{ error_report::ErrorReport, exit_code::ExitCode, summary_output::{ - ErrorSummary, UploadCommand, UploadReport, ValidateReport, + ErrorSummary, UploadCommand, UploadReport, ValidateReport, github, json::{self, Report}, text::{SummaryFileText, TextOptions, UploadText}, }, @@ -47,7 +47,9 @@ impl Cli { let token = match &self.command { Commands::Test(args) => args.token(), Commands::Upload(args) => args.token.clone(), - Commands::Validate(..) | Commands::SummaryOutputSchema => None, + Commands::Validate(..) + | Commands::SummaryOutputSchema + | Commands::GithubAnnotations(..) => None, }; token.map_or( @@ -62,6 +64,7 @@ impl Cli { Commands::Upload(..) => "upload", Commands::Validate(..) => "validate", Commands::SummaryOutputSchema => "summary-output-schema", + Commands::GithubAnnotations(..) => "github-annotations", } } @@ -74,7 +77,9 @@ impl Cli { match &self.command { Commands::Test(args) => args.org_url_slug(), Commands::Upload(args) => args.org_url_slug.clone(), - Commands::Validate(..) | Commands::SummaryOutputSchema => String::from("not used"), + Commands::Validate(..) + | Commands::SummaryOutputSchema + | Commands::GithubAnnotations(..) => String::from("not used"), } } @@ -82,7 +87,9 @@ impl Cli { match &self.command { Commands::Test(args) => args.summary_output_file(), Commands::Upload(args) => args.summary_output_file.clone(), - Commands::Validate(..) | Commands::SummaryOutputSchema => None, + Commands::Validate(..) + | Commands::SummaryOutputSchema + | Commands::GithubAnnotations(..) => None, } } @@ -90,7 +97,9 @@ impl Cli { let explicit_root = match &self.command { Commands::Test(args) => args.repo_root(), Commands::Upload(args) => args.repo_root.clone(), - Commands::Validate(..) | Commands::SummaryOutputSchema => None, + Commands::Validate(..) + | Commands::SummaryOutputSchema + | Commands::GithubAnnotations(..) => None, }; explicit_root .or(std::env::current_dir() @@ -114,6 +123,17 @@ enum Commands { Validate(ValidateArgs), /// Print the JSON Schema of the summary that `--summary-output-file` writes. SummaryOutputSchema, + /// Print GitHub Actions annotations for why a run failed, from its `--summary-output-file`. + /// Exits 0 only when it printed one. Used by the analytics-uploader action. + #[command(hide = true)] + GithubAnnotations(GithubAnnotationsArgs), +} + +#[derive(Debug, Args)] +struct GithubAnnotationsArgs { + /// The summary that `upload` or `test` wrote to `--summary-output-file`. + #[arg(long)] + summary_file: PathBuf, } fn close_out_and_exit( @@ -131,6 +151,12 @@ fn close_out_and_exit( // "the Sentry client must be initialized before starting an async runtime or spawning threads" // https://docs.sentry.io/platforms/rust/#async-main-function fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + // Before Sentry and logging start, so stdout carries only the workflow commands. + if let Commands::GithubAnnotations(args) = &cli.command { + std::process::exit(github::run(&args.summary_file).code()); + } + let release_name = format!("analytics-cli@{}", env!("CARGO_PKG_VERSION")); let guard = third_party::sentry::init(release_name.into(), None); @@ -138,7 +164,6 @@ fn main() -> anyhow::Result<()> { .enable_all() .build()? .block_on(async { - let cli = Cli::parse(); let org_url_slug = cli.org_url_slug(); let summary_output_file = cli.summary_output_file(); let log_level_filter = cli.verbose.log_level_filter(); @@ -301,6 +326,9 @@ async fn run(cli: Cli, render_sender: Sender) -> anyhow::Result< Ok(RunResult::Validate(result)) } Commands::SummaryOutputSchema => Ok(RunResult::SummaryOutputSchema), + Commands::GithubAnnotations(..) => { + anyhow::bail!("github-annotations runs before the async runtime starts") + } } } diff --git a/cli/src/summary_output/error.rs b/cli/src/summary_output/error.rs index 0985de3b..2b208e09 100644 --- a/cli/src/summary_output/error.rs +++ b/cli/src/summary_output/error.rs @@ -1,13 +1,13 @@ use api::client::get_api_host; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::{ error_report::{ErrorKind, ErrorReport, get_client_error_explanation, settings_url}, exit_code::ExitCode, }; -#[derive(Debug, Clone, Serialize, JsonSchema)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ErrorSummary { pub kind: ErrorKind, diff --git a/cli/src/summary_output/github.rs b/cli/src/summary_output/github.rs new file mode 100644 index 00000000..99f96fe2 --- /dev/null +++ b/cli/src/summary_output/github.rs @@ -0,0 +1,229 @@ +//! The hidden `github-annotations` command. It turns a summary file into GitHub Actions workflow +//! commands, so the analytics-uploader action can say why a run failed without reading the +//! summary's format itself. + +use std::{fmt, path::Path}; + +use pluralizer::pluralize; + +use super::{ + ErrorSummary, FailedTest, Outcome, Quarantining, UploadReport, + json::{self, Summary}, + text::{ + FAILED_WITHOUT_RUNNING_TESTS, FAILURE_MESSAGE_MAX_LINES, NO_TEST_FAILURES, NO_TESTS_FOUND, + QUARANTINE_LOOKUP_FAILED, SOME_TESTS_MAY_NOT_HAVE_RUN, UNAUTHORIZED_HINT, + }, +}; +use crate::{ + error_report::{CONNECTION_REFUSED_CONTEXT, ErrorKind, UNAUTHORIZED_CONTEXT}, + exit_code::ExitCode, +}; + +/// GitHub shows at most 10 error annotations per step, and the summary takes one of them. +const MAX_TEST_ANNOTATIONS: usize = 9; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Level { + Error, + Warning, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Annotation { + pub level: Level, + pub file: Option, + pub title: Option, + pub message: String, +} + +/// Prints the summary's annotations on stdout. The action passes them through when this exits +/// 0, and otherwise falls back to its own message, so anything else exits non-zero: a summary +/// with nothing to say, and one that can't be read. +pub fn run(summary_file: &Path) -> ExitCode { + let summary = match json::read_file(summary_file) { + Ok(summary) => summary, + Err(error) => { + eprintln!("{error:#}"); + return ExitCode::Failure; + } + }; + let annotations = annotations(&summary); + if annotations.is_empty() { + return ExitCode::Failure; + } + for annotation in annotations { + println!("{annotation}"); + } + ExitCode::Success +} + +pub fn annotations(summary: &Summary) -> Vec { + match summary { + Summary::Upload(report) => upload_annotations(report), + Summary::Error(error) => vec![Annotation::error(error_message(error))], + } +} + +/// Follows `text::render_upload`: an error that blocks quarantining is all there is to say, +/// and any other error is a warning before the outcome. +fn upload_annotations(report: &UploadReport) -> Vec { + let mut annotations = Vec::new(); + if let Some(error) = &report.error { + if error.kind.blocks_quarantining() { + return vec![Annotation::error(error_message(error))]; + } + annotations.push(Annotation::warning(error_message(error))); + } + + let exit_code = report.cli_exit_code(); + match report.outcome { + Outcome::NoTestsFound if !exit_code.is_success() => { + annotations.push(Annotation::error(format!( + "{NO_TESTS_FOUND}, but non zero exit code provided: {exit_code}\n{FAILED_WITHOUT_RUNNING_TESTS}" + ))); + } + Outcome::NoFailuresButNonZeroExitCode => { + annotations.push(Annotation::error(format!( + "{NO_TEST_FAILURES}, but non zero exit code provided: {exit_code}\n{SOME_TESTS_MAY_NOT_HAVE_RUN}" + ))); + } + Outcome::FailuresNotQuarantined => { + let tests = report.failures.iter().flat_map(|group| &group.tests); + annotations.push(Annotation::error(failures_message(report))); + annotations.extend(tests.take(MAX_TEST_ANNOTATIONS).map(test_annotation)); + } + Outcome::NoTestsFound | Outcome::NoFailures | Outcome::AllFailuresQuarantined => {} + } + annotations +} + +fn failures_message(report: &UploadReport) -> String { + let failed = report.counts.failed; + let mut message = format!( + "{} failed and {} not quarantined, using exit code: {}", + pluralize("test", failed as isize, true), + pluralize("was", failed as isize, false), + report.cli_exit_code(), + ); + let reason = match report.quarantining { + Quarantining::LookupFailed => Some(QUARANTINE_LOOKUP_FAILED), + Quarantining::DisabledViaCli => { + Some("Quarantining was turned off with --disable-quarantining") + } + Quarantining::DisabledViaApi => Some("Quarantining is turned off for this repo in Trunk"), + Quarantining::Checked | Quarantining::NotNeeded => None, + }; + if let Some(reason) = reason { + message.push_str(&format!("\n{reason}")); + } + if failed > MAX_TEST_ANNOTATIONS { + message.push_str(&format!( + "\nThe first {MAX_TEST_ANNOTATIONS} are annotated; see the log for the rest." + )); + } + message +} + +fn test_annotation(test: &FailedTest) -> Annotation { + let separator = if test.suite.is_empty() { "" } else { "/" }; + let mut lines: Vec<&str> = test + .failure_text + .as_deref() + .or(test.failure_message.as_deref()) + .map(|message| message.lines().take(FAILURE_MESSAGE_MAX_LINES).collect()) + .unwrap_or_default(); + if lines.is_empty() { + lines.push("Failed"); + } + if let Some(link) = &test.link { + lines.push(link); + } + Annotation { + level: Level::Error, + file: test.file.clone(), + title: Some(format!("{}{separator}{}", test.suite, test.name)), + message: lines.join("\n"), + } +} + +/// The same lines as `text::render_error`, without its exit code line. +fn error_message(error: &ErrorSummary) -> String { + let mut lines = Vec::new(); + match error.kind { + ErrorKind::ConnectionRefused => { + lines.extend(error.message.clone()); + lines.push(CONNECTION_REFUSED_CONTEXT.to_string()); + } + ErrorKind::Unauthorized => { + lines.extend(error.message.clone()); + lines.push(UNAUTHORIZED_CONTEXT.to_string()); + lines.extend( + error + .settings_url + .as_ref() + .map(|url| format!("{UNAUTHORIZED_HINT} {url}")), + ); + } + ErrorKind::GitRepository | ErrorKind::Interrupted | ErrorKind::Other => { + match &error.message { + Some(message) => { + lines.push(message.clone()); + lines.extend(error.explanation.clone()); + } + None => lines.push(error.error.clone()), + } + } + } + lines.join("\n") +} + +impl Annotation { + fn error(message: String) -> Self { + Self { + level: Level::Error, + file: None, + title: None, + message, + } + } + + fn warning(message: String) -> Self { + Self { + level: Level::Warning, + ..Self::error(message) + } + } +} + +/// A workflow command, escaped per GitHub's rules: +/// https://github.com/actions/toolkit/blob/main/packages/core/src/command.ts +impl fmt::Display for Annotation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let command = match self.level { + Level::Error => "error", + Level::Warning => "warning", + }; + let properties = [("file", &self.file), ("title", &self.title)] + .into_iter() + .filter_map(|(key, value)| Some(format!("{key}={}", escape_property(value.as_ref()?)))) + .collect::>(); + let separator = if properties.is_empty() { "" } else { " " }; + write!( + f, + "::{command}{separator}{}::{}", + properties.join(","), + escape_data(&self.message) + ) + } +} + +fn escape_data(value: &str) -> String { + value + .replace('%', "%25") + .replace('\r', "%0D") + .replace('\n', "%0A") +} + +fn escape_property(value: &str) -> String { + escape_data(value).replace(':', "%3A").replace(',', "%2C") +} diff --git a/cli/src/summary_output/json.rs b/cli/src/summary_output/json.rs index 04ee8b7b..c8f17510 100644 --- a/cli/src/summary_output/json.rs +++ b/cli/src/summary_output/json.rs @@ -1,7 +1,8 @@ use std::{fs, path::Path}; +use anyhow::Context; use schemars::{JsonSchema, Schema, generate::SchemaSettings}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use super::{ErrorSummary, UploadReport}; @@ -29,6 +30,20 @@ struct Versioned<'a> { report: Report<'a>, } +/// A summary read back from a file, owning its report. +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "type", rename_all = "SCREAMING_SNAKE_CASE")] +pub enum Summary { + Upload(Box), + Error(ErrorSummary), +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct SchemaVersion { + schema_version: u32, +} + pub fn schema() -> Schema { SchemaSettings::draft2020_12() .for_serialize() @@ -43,6 +58,25 @@ pub fn to_string(report: Report) -> serde_json::Result { }) } +/// Reads a summary that this or an older CLI wrote. A newer schema version is an error, since its +/// fields may have changed meaning. +pub fn from_str(json: &str) -> anyhow::Result { + let SchemaVersion { schema_version } = + serde_json::from_str(json).context("not a summary: no schemaVersion")?; + if schema_version > SCHEMA_VERSION { + anyhow::bail!( + "schemaVersion {schema_version} is newer than this CLI understands ({SCHEMA_VERSION})" + ); + } + Ok(serde_json::from_str(json)?) +} + +pub fn read_file(path: &Path) -> anyhow::Result { + let json = + fs::read_to_string(path).with_context(|| format!("couldn't read {}", path.display()))?; + from_str(&json).with_context(|| format!("couldn't read the summary in {}", path.display())) +} + /// Writes a sibling file and renames it into place, so a reader never sees half a summary. /// Not `tempfile::NamedTempFile`, which would leave the summary readable only by its owner. pub fn write_file(path: &Path, report: Report) -> anyhow::Result<()> { diff --git a/cli/src/summary_output/mod.rs b/cli/src/summary_output/mod.rs index 85e6b833..2270bec2 100644 --- a/cli/src/summary_output/mod.rs +++ b/cli/src/summary_output/mod.rs @@ -1,4 +1,5 @@ mod error; +pub mod github; pub mod json; mod notice; pub mod text; diff --git a/cli/src/summary_output/notice.rs b/cli/src/summary_output/notice.rs index 2da04e21..7a6ea973 100644 --- a/cli/src/summary_output/notice.rs +++ b/cli/src/summary_output/notice.rs @@ -1,14 +1,14 @@ use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct Notice { pub kind: NoticeKind, pub message: String, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum NoticeKind { /// A deprecated flag was passed. diff --git a/cli/src/summary_output/text.rs b/cli/src/summary_output/text.rs index 08d7596b..f58c71d1 100644 --- a/cli/src/summary_output/text.rs +++ b/cli/src/summary_output/text.rs @@ -94,6 +94,15 @@ impl EndOutput for SummaryFileText { } } +// Shared with the GitHub annotations, so both say the same thing. +pub(crate) const NO_TESTS_FOUND: &str = "No tests were found in the provided test results"; +pub(crate) const FAILED_WITHOUT_RUNNING_TESTS: &str = "The test step appears to have failed without running tests -- see above for the original failure."; +pub(crate) const NO_TEST_FAILURES: &str = "No test failures found"; +pub(crate) const SOME_TESTS_MAY_NOT_HAVE_RUN: &str = "This may indicate that some tests were not run or that there were other issues during the test run."; +pub(crate) const QUARANTINE_LOOKUP_FAILED: &str = "We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures"; +pub(crate) const FAILURE_MESSAGE_MAX_LINES: usize = 20; +pub(crate) const UNAUTHORIZED_HINT: &str = "Hint: You can find it under the settings page at"; + pub fn render_error(error: &ErrorSummary) -> anyhow::Result> { let mut lines = Vec::new(); lines.push(Line::from_iter([Span::new_styled( @@ -112,7 +121,7 @@ pub fn render_error(error: &ErrorSummary) -> anyhow::Result> { } ErrorKind::Unauthorized => { let mut hint = Line::from_iter([Span::new_unstyled_lossy(format!( - "Hint: You can find it under the settings page at {}", + "{UNAUTHORIZED_HINT} {}", error.settings_url.as_deref().unwrap_or_default() ))]); hint.pad_left(2); @@ -216,7 +225,9 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re ))?])); if report.quarantining == Quarantining::LookupFailed { output.push(Line::from_iter([Span::new_styled( - style(" We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures".to_string()).with(Color::Yellow).attribute(Attribute::Bold), + style(format!(" {QUARANTINE_LOOKUP_FAILED}")) + .with(Color::Yellow) + .attribute(Attribute::Bold), )?])); } output.push(Line::default()); @@ -270,22 +281,20 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re } match report.outcome { - Outcome::NoTestsFound if report.cli_exit_code().is_success() => { - output.push(Line::from_iter([Span::new_unstyled( - "⚠️ No tests were found in the provided test results", - )?])) - } + Outcome::NoTestsFound if report.cli_exit_code().is_success() => output.push( + Line::from_iter([Span::new_unstyled(format!("⚠️ {NO_TESTS_FOUND}"))?]), + ), Outcome::NoTestsFound => { output.push(Line::from_iter([ - Span::new_unstyled( - "⚠️ No tests were found in the provided test results, but non zero exit code provided: ", - )?, + Span::new_unstyled(format!( + "⚠️ {NO_TESTS_FOUND}, but non zero exit code provided: " + ))?, Span::new_styled( style(format!("{}", report.cli_exit_code())).attribute(Attribute::Bold), )?, ])); output.push(Line::from_iter([Span::new_unstyled( - "The test step appears to have failed without running tests -- see above for the original failure.", + FAILED_WITHOUT_RUNNING_TESTS, )?])); } Outcome::AllFailuresQuarantined => output.push(Line::from_iter([ @@ -299,15 +308,15 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re )?])), Outcome::NoFailuresButNonZeroExitCode => { output.push(Line::from_iter([ - Span::new_unstyled( - "⚠️ No test failures found, but non zero exit code provided: ", - )?, + Span::new_unstyled(format!( + "⚠️ {NO_TEST_FAILURES}, but non zero exit code provided: " + ))?, Span::new_styled( style(format!("{}", report.cli_exit_code())).attribute(Attribute::Bold), )?, ])); output.push(Line::from_iter([Span::new_unstyled( - "This may indicate that some tests were not run or that there were other issues during the test run.", + SOME_TESTS_MAY_NOT_HAVE_RUN, )?])); } Outcome::FailuresNotQuarantined => output.push(Line::from_iter([ @@ -384,7 +393,7 @@ fn render_groups(groups: &[FailureGroup], options: &TextOptions) -> anyhow::Resu fn render_failure_message(failure_message: &str) -> anyhow::Result> { let mut output = Vec::new(); let lines: Vec<&str> = failure_message.split('\n').collect(); - let max_lines = 20; + let max_lines = FAILURE_MESSAGE_MAX_LINES; let mut failure_header = Line::from_iter([Span::new_styled( style("Failure: ".to_string()).with(Color::DarkGrey), )?]); diff --git a/cli/src/summary_output/upload.rs b/cli/src/summary_output/upload.rs index 9810b216..1569e31f 100644 --- a/cli/src/summary_output/upload.rs +++ b/cli/src/summary_output/upload.rs @@ -4,14 +4,14 @@ use api::urls::{url_for_test_case, url_for_upload}; use bundle::Test; use context::junit::validator::JunitValidationLevel; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use super::{ErrorSummary, Notice}; use crate::{ exit_code::ExitCode, upload_command::UploadRunResult, validate_command::JunitReportValidations, }; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum UploadCommand { Upload, @@ -21,7 +21,7 @@ pub enum UploadCommand { RSpec, } -#[derive(Debug, Clone, Serialize, JsonSchema)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct UploadReport { /// Which command wrote this summary. @@ -49,7 +49,7 @@ pub struct UploadReport { pub notices: Vec, } -#[derive(Debug, Clone, Copy, PartialEq, Serialize, JsonSchema)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct TestCounts { pub total: usize, @@ -63,7 +63,7 @@ pub struct TestCounts { pub pass_ratio: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct FailureGroup { pub key: GroupKey, @@ -71,7 +71,7 @@ pub struct FailureGroup { } /// Groups sort by file, then by suite, then the rest, in declaration order. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema)] #[serde(tag = "type", content = "name", rename_all = "SCREAMING_SNAKE_CASE")] pub enum GroupKey { /// Failed tests from this file. @@ -82,7 +82,7 @@ pub enum GroupKey { Other, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct FailedTest { pub name: String, @@ -102,7 +102,7 @@ pub struct FailedTest { pub link: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct FileValidation { pub file: String, @@ -116,7 +116,7 @@ pub struct FileValidation { pub test_case_count: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ParseError { pub error: String, @@ -125,7 +125,7 @@ pub struct ParseError { } /// Whether the failed tests could be checked for quarantining, and if not, why. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum Quarantining { /// Trunk said which failed tests are quarantined. @@ -140,7 +140,7 @@ pub enum Quarantining { LookupFailed, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum Outcome { /// The test reports had no tests. diff --git a/cli/tests/summary/github.rs b/cli/tests/summary/github.rs new file mode 100644 index 00000000..79f3ae06 --- /dev/null +++ b/cli/tests/summary/github.rs @@ -0,0 +1,257 @@ +use std::fs; + +use api::client::ClientErrorExplanation; +use assert_cmd::{Command, cargo}; +use pretty_assertions::assert_eq; +use trunk_analytics_cli::{ + context_quarantine::QuarantineFetchStatus, + error_report::{ErrorReport, InterruptingError}, + exit_code::ExitCode, + summary_output::{ + ErrorSummary, UploadCommand, UploadReport, + github::annotations, + json::{self, Report}, + }, + upload_command::UploadRunResult, +}; + +use super::{ + unauthorized_error, + upload::{UPLOAD_ERROR_MESSAGE, failure, upload_result}, +}; + +/// Writes the summary as `upload` would, then reads it back the way the command does. +fn upload_annotations(result: &UploadRunResult) -> Vec { + let report = UploadReport::new(result, UploadCommand::Upload); + let summary = json::from_str(&json::to_string(Report::Upload(&report)).unwrap()).unwrap(); + annotations(&summary) + .iter() + .map(ToString::to_string) + .collect() +} + +fn error_annotations(report: &ErrorReport) -> Vec { + let error = ErrorSummary::new(report); + let summary = json::from_str(&json::to_string(Report::Error(&error)).unwrap()).unwrap(); + annotations(&summary) + .iter() + .map(ToString::to_string) + .collect() +} + +#[test] +fn reads_back_the_summary_it_wrote() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = ExitCode::Failure; + result.quarantine_context.failures = vec![failure("adds", "MathTests", Some("src/math.rs"))]; + let report = UploadReport::new(&result, UploadCommand::Upload); + let written = json::to_string(Report::Upload(&report)).unwrap(); + + let json::Summary::Upload(read) = json::from_str(&written).unwrap() else { + panic!("expected an upload summary"); + }; + assert_eq!(json::to_string(Report::Upload(&read)).unwrap(), written); +} + +#[test] +fn rejects_a_newer_schema_version() { + let error = json::from_str(r#"{"schemaVersion": 2, "type": "ERROR"}"#).unwrap_err(); + assert_eq!( + error.to_string(), + "schemaVersion 2 is newer than this CLI understands (1)" + ); + assert!(json::from_str("{}").is_err()); +} + +#[test] +fn says_nothing_about_a_passing_run() { + assert_eq!(upload_annotations(&upload_result(5)), Vec::::new()); +} + +#[test] +fn says_nothing_when_no_tests_were_found_with_exit_code_0() { + assert_eq!(upload_annotations(&upload_result(0)), Vec::::new()); +} + +#[test] +fn says_the_test_step_failed_without_running_tests() { + let mut result = upload_result(0); + result.quarantine_context.exit_code = ExitCode::Failure; + + assert_eq!( + upload_annotations(&result), + [ + "::error::No tests were found in the provided test results, but non zero exit code provided: 1%0AThe test step appears to have failed without running tests -- see above for the original failure." + ] + ); +} + +#[test] +fn says_some_tests_may_not_have_run() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = ExitCode::PassedThrough(2); + + assert_eq!( + upload_annotations(&result), + [ + "::error::No test failures found, but non zero exit code provided: 2%0AThis may indicate that some tests were not run or that there were other issues during the test run." + ] + ); +} + +#[test] +fn annotates_each_failure_that_was_not_quarantined() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = ExitCode::Failure; + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![failure("adds", "MathTests", Some("src/math.rs"))]; + let mut divides = failure("divides", "MathTests", None); + divides.failure_message = Some(String::from("expected 1, got 2\n100% wrong")); + result.quarantine_context.failures = vec![ + failure("subtracts", "Math, Tests: unit", Some("src/math.rs")), + divides, + ]; + + assert_eq!( + upload_annotations(&result), + [ + "::error::2 tests failed and were not quarantined, using exit code: 1", + "::error file=src/math.rs,title=Math%2C Tests%3A unit/subtracts::Failed%0Ahttps://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%252Fanalytics-cli", + "::error title=MathTests/divides::expected 1, got 2%0A100%25 wrong%0Ahttps://app.trunk.io/test-org/flaky-tests/test/divides-id?repo=trunk-io%252Fanalytics-cli", + ] + ); +} + +#[test] +fn says_why_failures_could_not_be_quarantined() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = ExitCode::Failure; + result.quarantine_context.fetch_status = + QuarantineFetchStatus::FetchFailed(anyhow::anyhow!("request timed out")); + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + + assert_eq!( + upload_annotations(&result)[0], + "::error::1 test failed and was not quarantined, using exit code: 1%0AWe were unable to determine the quarantine status for tests. Any failing tests will be reported as failures" + ); + + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result.disable_quarantining = true; + assert_eq!( + upload_annotations(&result)[0], + "::error::1 test failed and was not quarantined, using exit code: 1%0AQuarantining was turned off with --disable-quarantining" + ); +} + +#[test] +fn annotates_at_most_nine_failures() { + let mut result = upload_result(12); + result.quarantine_context.exit_code = ExitCode::Failure; + result.quarantine_context.failures = (0..12) + .map(|i| failure(&format!("test{i}"), "MathTests", None)) + .collect(); + + let annotations = upload_annotations(&result); + assert_eq!(annotations.len(), 10); + assert_eq!( + annotations[0], + "::error::12 tests failed and were not quarantined, using exit code: 1%0AThe first 9 are annotated; see the log for the rest." + ); +} + +#[test] +fn says_only_the_error_when_it_blocks_quarantining() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = ExitCode::Failure; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + unauthorized_error(), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_annotations(&result), + [ + "::error::There was an unexpected error that occurred while uploading test results%0AUnathorized access, your Trunk organization URL slug or token may be incorrect%0AHint: You can find it under the settings page at https://app.trunk.io/test-org/settings" + ] + ); +} + +#[test] +fn warns_about_an_upload_error_before_the_failures() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = ExitCode::Failure; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + anyhow::anyhow!("HTTP status client error (400 Bad Request)") + .context(ClientErrorExplanation { + code: Some(String::from("SOME_CODE")), + message: Some(String::from("Pass --test-collection-id.")), + }) + .context("Error in create bundle upload endpoint"), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_annotations(&result)[..2].to_vec(), + [ + "::warning::There was an unexpected error that occurred while uploading test results%0APass --test-collection-id.", + "::error::1 test failed and was not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn annotates_a_run_that_failed_outright() { + let report = ErrorReport::new( + anyhow::Error::from(InterruptingError::new(String::from( + "No test output files found to upload.", + ))), + String::from("test-org"), + None, + ); + + assert_eq!( + error_annotations(&report), + ["::error::No test output files found to upload."] + ); +} + +fn github_annotations(summary_file: &std::path::Path) -> assert_cmd::assert::Assert { + Command::new(cargo::cargo_bin!()) + .args(["github-annotations", "--summary-file"]) + .arg(summary_file) + .assert() +} + +#[test] +fn the_command_prints_annotations_and_exits_0() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("summary.json"); + let mut result = upload_result(0); + result.quarantine_context.exit_code = ExitCode::Failure; + let report = UploadReport::new(&result, UploadCommand::Upload); + json::write_file(&path, Report::Upload(&report)).unwrap(); + + github_annotations(&path).success().stdout( + "::error::No tests were found in the provided test results, but non zero exit code provided: 1%0AThe test step appears to have failed without running tests -- see above for the original failure.\n", + ); +} + +#[test] +fn the_command_exits_non_zero_with_nothing_to_say() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("summary.json"); + let report = UploadReport::new(&upload_result(5), UploadCommand::Upload); + json::write_file(&path, Report::Upload(&report)).unwrap(); + + github_annotations(&path).code(1).stdout(""); + + fs::remove_file(&path).unwrap(); + github_annotations(&path).code(1).stdout(""); +} diff --git a/cli/tests/summary/main.rs b/cli/tests/summary/main.rs index 07ecdf68..5b5c3eda 100644 --- a/cli/tests/summary/main.rs +++ b/cli/tests/summary/main.rs @@ -1,4 +1,5 @@ mod error_report; +mod github; mod schema; mod upload; mod validate; diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index a522cdf8..3c10add0 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -32,7 +32,7 @@ use trunk_analytics_cli::{ use crate::unauthorized_error; const BUNDLE_UPLOAD_ID: &str = "4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d"; -const UPLOAD_ERROR_MESSAGE: &str = +pub(super) const UPLOAD_ERROR_MESSAGE: &str = "There was an unexpected error that occurred while uploading test results"; fn upload_summary(result: &UploadRunResult) -> Vec { @@ -52,7 +52,7 @@ fn repo() -> RepoUrlParts { } } -fn upload_result(num_tests: usize) -> UploadRunResult { +pub(super) fn upload_result(num_tests: usize) -> UploadRunResult { let mut quarantine_context = QuarantineContext::skip_fetch(Vec::new()); quarantine_context.repo = repo(); quarantine_context.org_url_slug = String::from("test-org"); @@ -117,7 +117,7 @@ fn in_collection(mut result: UploadRunResult) -> UploadRunResult { result } -fn failure(name: &str, parent_name: &str, file: Option<&str>) -> Test { +pub(super) fn failure(name: &str, parent_name: &str, file: Option<&str>) -> Test { Test { name: String::from(name), parent_name: String::from(parent_name), From 0b056b4eaba3e190e875057251963c71eb50c855 Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 28 Oct 2026 16:11:18 +0000 Subject: [PATCH 19/19] refactor(cli): make the summary's JSON helpers methods, and read newer schemas that still parse - Report::{schema, to_json, write_file}, and Summary::read_file plus FromStr, replace json::* free functions. - A newer schemaVersion only fails when the summary doesn't parse; one that does is read with a warning. - An unknown error kind from a newer CLI reads as OTHER instead of failing the whole summary. Co-Authored-By: Claude Opus 5.5 --- cli/src/error_report.rs | 2 + cli/src/main.rs | 4 +- cli/src/summary_output/github.rs | 4 +- cli/src/summary_output/json.rs | 115 ++++++++++++++++++------------ cli/tests/summary/error_report.rs | 4 +- cli/tests/summary/github.rs | 49 ++++++++++--- cli/tests/summary/schema.rs | 2 +- cli/tests/summary/upload.rs | 2 +- test_report/src/report.rs | 2 +- 9 files changed, 117 insertions(+), 67 deletions(-) diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index 130540b5..c0dc894b 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -105,6 +105,8 @@ pub enum ErrorKind { GitRepository, /// The run stopped early, such as when it found no test reports and empty results aren't allowed. Interrupted, + /// Also what a kind from a newer CLI reads as. + #[serde(other)] Other, } diff --git a/cli/src/main.rs b/cli/src/main.rs index 425ef7dc..80227220 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -247,7 +247,7 @@ fn main() -> anyhow::Result<()> { close_out_and_exit(exit_code, guard, render_sender, render_handle) } MainOutcome::Finished(Ok(RunResult::SummaryOutputSchema)) => { - println!("{}", serde_json::to_string_pretty(&json::schema())?); + println!("{}", serde_json::to_string_pretty(&json::Report::schema())?); close_out_and_exit(ExitCode::Success, guard, render_sender, render_handle) } MainOutcome::Finished(Err(error)) => { @@ -276,7 +276,7 @@ fn write_summary_file(path: Option<&Path>, report: Report, render_sender: &Sende let Some(path) = path else { return; }; - let text = SummaryFileText::new(path, json::write_file(path, report)); + let text = SummaryFileText::new(path, report.write_file(path)); send_message( DisplayMessage::Final(Arc::new(text), String::from("summary file display")), render_sender, diff --git a/cli/src/summary_output/github.rs b/cli/src/summary_output/github.rs index 99f96fe2..eeee6a16 100644 --- a/cli/src/summary_output/github.rs +++ b/cli/src/summary_output/github.rs @@ -8,7 +8,7 @@ use pluralizer::pluralize; use super::{ ErrorSummary, FailedTest, Outcome, Quarantining, UploadReport, - json::{self, Summary}, + json::Summary, text::{ FAILED_WITHOUT_RUNNING_TESTS, FAILURE_MESSAGE_MAX_LINES, NO_TEST_FAILURES, NO_TESTS_FOUND, QUARANTINE_LOOKUP_FAILED, SOME_TESTS_MAY_NOT_HAVE_RUN, UNAUTHORIZED_HINT, @@ -40,7 +40,7 @@ pub struct Annotation { /// 0, and otherwise falls back to its own message, so anything else exits non-zero: a summary /// with nothing to say, and one that can't be read. pub fn run(summary_file: &Path) -> ExitCode { - let summary = match json::read_file(summary_file) { + let summary = match Summary::read_file(summary_file) { Ok(summary) => summary, Err(error) => { eprintln!("{error:#}"); diff --git a/cli/src/summary_output/json.rs b/cli/src/summary_output/json.rs index c8f17510..c7915bed 100644 --- a/cli/src/summary_output/json.rs +++ b/cli/src/summary_output/json.rs @@ -1,4 +1,4 @@ -use std::{fs, path::Path}; +use std::{fs, path::Path, str::FromStr}; use anyhow::Context; use schemars::{JsonSchema, Schema, generate::SchemaSettings}; @@ -44,58 +44,79 @@ struct SchemaVersion { schema_version: u32, } -pub fn schema() -> Schema { - SchemaSettings::draft2020_12() - .for_serialize() - .into_generator() - .into_root_schema_for::>() -} +impl Report<'_> { + /// What `summary-output-schema` prints. + pub fn schema() -> Schema { + SchemaSettings::draft2020_12() + .for_serialize() + .into_generator() + .into_root_schema_for::>() + } -pub fn to_string(report: Report) -> serde_json::Result { - serde_json::to_string_pretty(&Versioned { - schema_version: SCHEMA_VERSION, - report, - }) -} + pub fn to_json(self) -> serde_json::Result { + serde_json::to_string_pretty(&Versioned { + schema_version: SCHEMA_VERSION, + report: self, + }) + } -/// Reads a summary that this or an older CLI wrote. A newer schema version is an error, since its -/// fields may have changed meaning. -pub fn from_str(json: &str) -> anyhow::Result { - let SchemaVersion { schema_version } = - serde_json::from_str(json).context("not a summary: no schemaVersion")?; - if schema_version > SCHEMA_VERSION { - anyhow::bail!( - "schemaVersion {schema_version} is newer than this CLI understands ({SCHEMA_VERSION})" - ); + /// Writes a sibling file and renames it into place, so a reader never sees half a summary. + /// Not `tempfile::NamedTempFile`, which would leave the summary readable only by its owner. + pub fn write_file(self, path: &Path) -> anyhow::Result<()> { + let json = self.to_json()?; + let file_name = path + .file_name() + .ok_or_else(|| anyhow::anyhow!("{} is not a file path", path.display()))?; + let dir = path.parent().unwrap_or(Path::new("")); + if !dir.as_os_str().is_empty() { + fs::create_dir_all(dir)?; + } + let temp_path = dir.join(format!( + ".{}.{}.tmp", + file_name.to_string_lossy(), + std::process::id() + )); + let written = fs::write(&temp_path, json).and_then(|()| fs::rename(&temp_path, path)); + if written.is_err() { + let _ = fs::remove_file(&temp_path); + } + Ok(written?) } - Ok(serde_json::from_str(json)?) } -pub fn read_file(path: &Path) -> anyhow::Result { - let json = - fs::read_to_string(path).with_context(|| format!("couldn't read {}", path.display()))?; - from_str(&json).with_context(|| format!("couldn't read the summary in {}", path.display())) +impl Summary { + pub fn read_file(path: &Path) -> anyhow::Result { + let json = fs::read_to_string(path) + .with_context(|| format!("couldn't read {}", path.display()))?; + json.parse() + .with_context(|| format!("couldn't read the summary in {}", path.display())) + } } -/// Writes a sibling file and renames it into place, so a reader never sees half a summary. -/// Not `tempfile::NamedTempFile`, which would leave the summary readable only by its owner. -pub fn write_file(path: &Path, report: Report) -> anyhow::Result<()> { - let json = to_string(report)?; - let file_name = path - .file_name() - .ok_or_else(|| anyhow::anyhow!("{} is not a file path", path.display()))?; - let dir = path.parent().unwrap_or(Path::new("")); - if !dir.as_os_str().is_empty() { - fs::create_dir_all(dir)?; - } - let temp_path = dir.join(format!( - ".{}.{}.tmp", - file_name.to_string_lossy(), - std::process::id() - )); - let written = fs::write(&temp_path, json).and_then(|()| fs::rename(&temp_path, path)); - if written.is_err() { - let _ = fs::remove_file(&temp_path); +impl FromStr for Summary { + type Err = anyhow::Error; + + /// Reads a summary that any CLI wrote. A newer schema version is only an error when the + /// summary doesn't parse; one that does, such as one with only new fields, is read with a + /// warning. + fn from_str(json: &str) -> anyhow::Result { + let SchemaVersion { schema_version } = + serde_json::from_str(json).context("not a summary: no schemaVersion")?; + let newer = schema_version > SCHEMA_VERSION; + let newer_message = || { + format!( + "schemaVersion {schema_version} is newer than this CLI understands ({SCHEMA_VERSION})" + ) + }; + match serde_json::from_str(json) { + Ok(summary) => { + if newer { + tracing::warn!("{}; reading it anyway", newer_message()); + } + Ok(summary) + } + Err(error) if newer => Err(anyhow::Error::from(error).context(newer_message())), + Err(error) => Err(error.into()), + } } - Ok(written?) } diff --git a/cli/tests/summary/error_report.rs b/cli/tests/summary/error_report.rs index 8340e6a4..4475f518 100644 --- a/cli/tests/summary/error_report.rs +++ b/cli/tests/summary/error_report.rs @@ -199,7 +199,7 @@ fn writes_an_error_as_json() { let error = ErrorSummary::new(&report); assert_eq!( - serde_json::from_str::(&json::to_string(Report::Error(&error)).unwrap()) + serde_json::from_str::(&Report::Error(&error).to_json().unwrap()) .unwrap(), serde_json::json!({ "schemaVersion": 1, @@ -220,7 +220,7 @@ fn writes_the_settings_url_and_exit_code_of_an_unauthorized_error_as_json() { let report = ErrorReport::new(unauthorized_error(), String::from("fake-org-slug"), None); let error = ErrorSummary::new(&report); let json: serde_json::Value = - serde_json::from_str(&json::to_string(Report::Error(&error)).unwrap()).unwrap(); + serde_json::from_str(&Report::Error(&error).to_json().unwrap()).unwrap(); assert_eq!(json["kind"], "UNAUTHORIZED"); assert_eq!( diff --git a/cli/tests/summary/github.rs b/cli/tests/summary/github.rs index 79f3ae06..a9823277 100644 --- a/cli/tests/summary/github.rs +++ b/cli/tests/summary/github.rs @@ -5,7 +5,7 @@ use assert_cmd::{Command, cargo}; use pretty_assertions::assert_eq; use trunk_analytics_cli::{ context_quarantine::QuarantineFetchStatus, - error_report::{ErrorReport, InterruptingError}, + error_report::{ErrorKind, ErrorReport, InterruptingError}, exit_code::ExitCode, summary_output::{ ErrorSummary, UploadCommand, UploadReport, @@ -23,7 +23,11 @@ use super::{ /// Writes the summary as `upload` would, then reads it back the way the command does. fn upload_annotations(result: &UploadRunResult) -> Vec { let report = UploadReport::new(result, UploadCommand::Upload); - let summary = json::from_str(&json::to_string(Report::Upload(&report)).unwrap()).unwrap(); + let summary = Report::Upload(&report) + .to_json() + .unwrap() + .parse::() + .unwrap(); annotations(&summary) .iter() .map(ToString::to_string) @@ -32,7 +36,11 @@ fn upload_annotations(result: &UploadRunResult) -> Vec { fn error_annotations(report: &ErrorReport) -> Vec { let error = ErrorSummary::new(report); - let summary = json::from_str(&json::to_string(Report::Error(&error)).unwrap()).unwrap(); + let summary = Report::Error(&error) + .to_json() + .unwrap() + .parse::() + .unwrap(); annotations(&summary) .iter() .map(ToString::to_string) @@ -45,22 +53,41 @@ fn reads_back_the_summary_it_wrote() { result.quarantine_context.exit_code = ExitCode::Failure; result.quarantine_context.failures = vec![failure("adds", "MathTests", Some("src/math.rs"))]; let report = UploadReport::new(&result, UploadCommand::Upload); - let written = json::to_string(Report::Upload(&report)).unwrap(); + let written = Report::Upload(&report).to_json().unwrap(); - let json::Summary::Upload(read) = json::from_str(&written).unwrap() else { + let json::Summary::Upload(read) = written.parse().unwrap() else { panic!("expected an upload summary"); }; - assert_eq!(json::to_string(Report::Upload(&read)).unwrap(), written); + assert_eq!(Report::Upload(&read).to_json().unwrap(), written); } #[test] fn rejects_a_newer_schema_version() { - let error = json::from_str(r#"{"schemaVersion": 2, "type": "ERROR"}"#).unwrap_err(); + let error = r#"{"schemaVersion": 2, "type": "ERROR"}"#.parse::().unwrap_err(); assert_eq!( error.to_string(), "schemaVersion 2 is newer than this CLI understands (1)" ); - assert!(json::from_str("{}").is_err()); + assert!("{}".parse::().is_err()); +} + +#[test] +fn reads_a_newer_schema_version_that_still_parses() { + let error = ErrorSummary::new(&ErrorReport::new( + unauthorized_error(), + String::from("fake-org-slug"), + None, + )); + let mut written: serde_json::Value = + serde_json::from_str(&Report::Error(&error).to_json().unwrap()).unwrap(); + written["schemaVersion"] = serde_json::json!(2); + written["kind"] = serde_json::json!("SOME_NEW_KIND"); + written["someNewField"] = serde_json::json!(true); + + let json::Summary::Error(read) = written.to_string().parse().unwrap() else { + panic!("expected an error summary"); + }; + assert_eq!(read.kind, ErrorKind::Other); } #[test] @@ -109,7 +136,7 @@ fn annotates_each_failure_that_was_not_quarantined() { .quarantine_status .quarantine_results = vec![failure("adds", "MathTests", Some("src/math.rs"))]; let mut divides = failure("divides", "MathTests", None); - divides.failure_message = Some(String::from("expected 1, got 2\n100% wrong")); + divides.failure_text = Some(String::from("expected 1, got 2\n100% wrong")); result.quarantine_context.failures = vec![ failure("subtracts", "Math, Tests: unit", Some("src/math.rs")), divides, @@ -236,7 +263,7 @@ fn the_command_prints_annotations_and_exits_0() { let mut result = upload_result(0); result.quarantine_context.exit_code = ExitCode::Failure; let report = UploadReport::new(&result, UploadCommand::Upload); - json::write_file(&path, Report::Upload(&report)).unwrap(); + Report::Upload(&report).write_file(&path).unwrap(); github_annotations(&path).success().stdout( "::error::No tests were found in the provided test results, but non zero exit code provided: 1%0AThe test step appears to have failed without running tests -- see above for the original failure.\n", @@ -248,7 +275,7 @@ fn the_command_exits_non_zero_with_nothing_to_say() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("summary.json"); let report = UploadReport::new(&upload_result(5), UploadCommand::Upload); - json::write_file(&path, Report::Upload(&report)).unwrap(); + Report::Upload(&report).write_file(&path).unwrap(); github_annotations(&path).code(1).stdout(""); diff --git a/cli/tests/summary/schema.rs b/cli/tests/summary/schema.rs index 4606a1bc..7cfbc3e4 100644 --- a/cli/tests/summary/schema.rs +++ b/cli/tests/summary/schema.rs @@ -14,6 +14,6 @@ fn prints_the_summary_output_schema() { assert_eq!( serde_json::from_slice::(&output).unwrap(), - serde_json::to_value(json::schema()).unwrap() + serde_json::to_value(json::Report::schema()).unwrap() ); } diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs index 3c10add0..e3558fb9 100644 --- a/cli/tests/summary/upload.rs +++ b/cli/tests/summary/upload.rs @@ -887,7 +887,7 @@ fn exits_with_the_test_run_exit_code_when_an_error_has_none() { fn upload_json(result: &UploadRunResult) -> serde_json::Value { let report = UploadReport::new(result, UploadCommand::Upload); - serde_json::from_str(&json::to_string(Report::Upload(&report)).unwrap()).unwrap() + serde_json::from_str(&Report::Upload(&report).to_json().unwrap()).unwrap() } #[test] diff --git a/test_report/src/report.rs b/test_report/src/report.rs index 75b7b022..68361ccf 100644 --- a/test_report/src/report.rs +++ b/test_report/src/report.rs @@ -1146,7 +1146,7 @@ impl MutTestReport { fn write_summary_file(path: &Path, report: Report) { eprintln!( "{}", - SummaryFileText::new(path, json::write_file(path, report)).message() + SummaryFileText::new(path, report.write_file(path)).message() ); }