From 3d552ee1d920ff898e4438848682b702a4c9ee2c Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 21 Oct 2026 22:54:30 +0000 Subject: [PATCH 1/2] test(cli): lock down the current summary output Adds tests that compare the plain-text summary line by line for upload, validate and ErrorReport, so the summary-as-data refactors (TRUNK-22401, TRUNK-22402) can show they change nothing. Each type goes through one helper (upload_summary, validate_summary, error_summary), which is the only thing those refactors should need to change. Refs TRUNK-22399 Co-Authored-By: Claude Opus 5.5 (1M context) --- cli/src/error_report.rs | 166 +++++++- cli/src/upload_command.rs | 814 +++++++++++++++++++++++++++++++++++- cli/src/validate_command.rs | 189 ++++++++- 3 files changed, 1159 insertions(+), 10 deletions(-) diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index e26bbc60..98ea9b20 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -240,6 +240,32 @@ impl EndOutput for ErrorReport { } } +#[cfg(test)] +fn error_summary(report: &ErrorReport) -> Vec { + report + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() +} + +#[cfg(test)] +pub(crate) fn unauthorized_error() -> anyhow::Error { + let response = http::Response::builder().status(401).body("").unwrap(); + anyhow::Error::from( + reqwest::Response::from(response) + .error_for_status() + .unwrap_err(), + ) +} + +#[cfg(test)] +fn connection_refused_error() -> anyhow::Error { + anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)) + .context("Failed to create bundle upload") +} + #[test] fn adds_settings_if_domain_present() { let host = "https://api.fake-trunk.io"; @@ -264,12 +290,7 @@ fn prints_a_client_error_explanation_beneath_the_base_message() { Some(String::from("There was an unexpected error")), ); - let text = report - .output() - .unwrap() - .iter() - .map(|line| line.to_unstyled()) - .collect::>(); + let text = error_summary(&report); assert_eq!( text[..4], @@ -281,3 +302,136 @@ fn prints_a_client_error_explanation_beneath_the_base_message() { ] ); } + +#[test] +fn prints_connection_refused_beneath_the_base_message() { + let report = ErrorReport::new( + connection_refused_error(), + String::from("fake-org-slug"), + Some(String::from("There was an unexpected error")), + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "There was an unexpected error", + "", + "Unable to connect to trunk's server", + ] + ); +} + +#[test] +fn prints_connection_refused_without_a_base_message() { + let report = ErrorReport::new( + connection_refused_error(), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + ["Error Encountered", "Unable to connect to trunk's server"] + ); +} + +#[test] +fn prints_unauthorized_with_a_settings_hint() { + let report = ErrorReport::new(unauthorized_error(), String::from("fake-org-slug"), None); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "", + "", + "Unathorized access, your Trunk organization URL slug or token may be incorrect", + " Hint: You can find it under the settings page at https://app.trunk.io/fake-org-slug/settings", + ] + ); +} + +#[test] +fn prints_a_generic_error_without_a_base_message() { + let report = ErrorReport::new( + anyhow::anyhow!("--allow-forked-pr-uploads requires --test-collection-id"), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "--allow-forked-pr-uploads requires --test-collection-id", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + ] + ); +} + +#[test] +fn prints_an_interrupting_error_with_its_exit_code() { + let report = ErrorReport::new( + anyhow::anyhow!(InterruptingError::new( + "No test output files found to validate" + )), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "No test output files found to validate", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, returning exit code: 70", + ] + ); +} + +#[test] +fn prints_the_default_exit_code() { + let mut report = ErrorReport::new( + anyhow::anyhow!("something went wrong"), + String::from("fake-org-slug"), + None, + ); + report.context.exit_code = Some(exitcode::OK); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "something went wrong", + "For more help, contact us at https://slack.trunk.io/", + "", + "No errors occurred, returning default exit code: 0", + ] + ); +} + +#[test] +fn prints_any_other_exit_code() { + let mut report = ErrorReport::new( + anyhow::anyhow!("something went wrong"), + String::from("fake-org-slug"), + None, + ); + report.context.exit_code = Some(exitcode::USAGE); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "something went wrong", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, returning exit code: 64", + ] + ); +} diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index faa0a33b..9e35fc2b 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -1288,9 +1288,819 @@ impl EndOutput for UploadRunResult { #[cfg(test)] mod tests { - use superconsole::{Line, style::Attribute}; + use std::collections::{BTreeMap, HashMap}; - use super::url_span; + use api::{client::ClientErrorExplanation, urls::TestCaseGuidScope}; + use bundle::{ + BundleMetaBaseProps, BundleMetaDebugProps, BundleMetaJunitProps, TestCollectionProps, + }; + use context::{ + junit::validator::{ + JunitReportValidation, JunitReportValidationIssueSubOptimal, + JunitTestCaseValidationIssueInvalid, JunitTestCaseValidationIssueSubOptimal, + JunitValidationIssue, JunitValidationIssueType, + }, + repo::{BundleRepo, RepoUrlParts}, + }; + use pretty_assertions::assert_eq; + + use super::*; + use crate::{context_quarantine::QuarantineFetchStatus, error_report::unauthorized_error}; + + const BUNDLE_UPLOAD_ID: &str = "4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d"; + const UPLOAD_ERROR_MESSAGE: &str = + "There was an unexpected error that occurred while uploading test results"; + + fn upload_summary(result: &UploadRunResult) -> Vec { + result + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() + } + + fn repo() -> RepoUrlParts { + RepoUrlParts { + host: String::from("github.com"), + owner: String::from("trunk-io"), + name: String::from("analytics-cli"), + } + } + + fn upload_result(num_tests: usize) -> UploadRunResult { + let mut quarantine_context = QuarantineContext::skip_fetch(Vec::new()); + quarantine_context.repo = repo(); + quarantine_context.org_url_slug = String::from("test-org"); + UploadRunResult { + error_report: None, + quarantine_context, + meta: BundleMeta { + base_props: BundleMetaBaseProps { + version: String::new(), + cli_version: String::new(), + org: String::from("test-org"), + test_collection: None, + repo: BundleRepo { + repo: repo(), + ..Default::default() + }, + bundle_upload_id: String::from(BUNDLE_UPLOAD_ID), + tags: Vec::new(), + file_sets: Vec::new(), + envs: HashMap::new(), + upload_time_epoch: 0, + test_command: None, + os_info: None, + quarantined_tests: Vec::new(), + codeowners: None, + use_uncloned_repo: None, + quarantine_resolution_mode: QuarantineResolutionMode::default(), + }, + junit_props: BundleMetaJunitProps { + num_files: 1, + num_tests, + }, + debug_props: BundleMetaDebugProps { + command_line: String::new(), + trunk_envs: HashMap::new(), + }, + bundle_upload_id_v2: String::new(), + variant: None, + internal_bundled_file: None, + failed_tests: Vec::new(), + }, + validations: JunitReportValidations::new(BTreeMap::new()), + validation_report: ValidationReport::Limited, + show_failure_messages: false, + test_collection_short_id: None, + hide_test_collection_links: false, + api_address: String::from("https://api.trunk.io"), + guid_scope: None, + } + } + + fn in_collection(mut result: UploadRunResult) -> UploadRunResult { + result.test_collection_short_id = Some(String::from("abc123")); + result.meta.base_props.test_collection = Some(TestCollectionProps { + short_id: String::from("abc123"), + bundle_meta_id: String::from("0b6f4c1e-8a7d-4e2b-9c3f-5d1a2b3c4d5e"), + bundle_meta_created_at: String::from("2026-10-21T12:00:00Z"), + }); + result + } + + fn failure(name: &str, parent_name: &str, file: Option<&str>) -> Test { + Test { + name: String::from(name), + parent_name: String::from(parent_name), + class_name: None, + file: file.map(String::from), + id: format!("{name}-id"), + timestamp_millis: None, + is_quarantined: false, + failure_message: None, + variant: None, + } + } + + fn warning(issue: JunitTestCaseValidationIssueSubOptimal) -> JunitValidationIssueType { + JunitValidationIssueType::TestCase(JunitValidationIssue::SubOptimal(issue)) + } + + fn report_warning(issue: JunitReportValidationIssueSubOptimal) -> JunitValidationIssueType { + JunitValidationIssueType::Report(JunitValidationIssue::SubOptimal(issue)) + } + + fn name_too_short() -> JunitValidationIssueType { + JunitValidationIssueType::TestCase(JunitValidationIssue::Invalid( + JunitTestCaseValidationIssueInvalid::TestCaseNameTooShort(String::new()), + )) + } + + fn validation( + all_issues: Vec, + ) -> anyhow::Result { + Ok(JunitReportValidation { + all_issues, + ..Default::default() + }) + } + + fn validations() -> JunitReportValidations { + JunitReportValidations::new(BTreeMap::from([ + ( + String::from("test-results/a.xml"), + validation(vec![ + name_too_short(), + name_too_short(), + report_warning( + JunitReportValidationIssueSubOptimal::TestCasesFileOrFilepathMissing, + ), + report_warning(JunitReportValidationIssueSubOptimal::MissingTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::FutureTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::OldTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::StaleTimestamps), + warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration), + warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseClassnameTooShort( + String::new(), + ), + ), + warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNameTooLong( + String::new(), + )), + warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseFileOrFilepathTooLong( + String::new(), + ), + ), + ]), + ), + ( + String::from("test-results/b.xml"), + Err(anyhow::anyhow!("unexpected end of file")), + ), + ( + String::from("test-results/c.xml"), + validation(vec![name_too_short()]), + ), + ( + String::from("test-results/d.xml"), + validation(vec![warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration, + )]), + ), + (String::from("test-results/e.xml"), validation(Vec::new())), + (String::from("test-results/f.xml"), validation(Vec::new())), + (String::from("test-results/g.xml"), validation(Vec::new())), + (String::from("test-results/h.xml"), validation(Vec::new())), + ( + String::from("test-results/i.xml"), + validation(vec![report_warning( + JunitReportValidationIssueSubOptimal::MissingTimestamps, + )]), + ), + ])) + } + + #[test] + fn prints_a_passing_run() { + let result = upload_result(5); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_that_no_tests_were_found() { + let result = upload_result(0); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 0 Pass: 0 Fail: 0 Quarantined: 0 Pass Ratio: N/A", + "", + "⚠️ No tests were found in the provided test results", + ] + ); + } + + #[test] + fn prints_failures_that_were_all_quarantined() { + let mut result = upload_result(4); + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .group_is_quarantined = true; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![ + failure("adds", "MathTests", Some("src/math.rs")), + failure("subtracts", "MathTests", Some("src/math.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 4 Pass: 2 Fail: 0 Quarantined: 2 Pass Ratio: 50.0%", + "", + "β€οΈβ€πŸ©Ή 2 tests failed and were quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success (0)", + ] + ); + } + + #[test] + fn prints_failures_that_were_not_quarantined() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![failure("adds", "MathTests", Some("src/math.rs"))]; + result.quarantine_context.failures = vec![ + failure("subtracts", "MathTests", Some("src/math.rs")), + failure("divides", "MathTests", Some("src/math.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 2 Fail: 2 Quarantined: 1 Pass Ratio: 40.0%", + "", + "β€οΈβ€πŸ©Ή 1 test failed and was quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "❌ 2 tests failed and were not quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/divides", + " β€· https://app.trunk.io/test-org/flaky-tests/test/divides-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn prints_a_warning_when_the_quarantine_lookup_failed() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.quarantine_context.fetch_status = + QuarantineFetchStatus::FetchFailed(anyhow::anyhow!("request timed out")); + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", + " We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn stops_at_an_error_that_blocks_quarantining() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + unauthorized_error(), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_summary(&result), + [ + "", + "Error Encountered", + "There was an unexpected error that occurred while uploading test results", + "", + "Unathorized access, your Trunk organization URL slug or token may be incorrect", + " Hint: You can find it under the settings page at https://app.trunk.io/test-org/settings", + ] + ); + } + + #[test] + fn proceeds_past_an_upload_error() { + let mut result = upload_result(3); + result.meta.base_props.bundle_upload_id = String::new(); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + anyhow::anyhow!("HTTP status client error (400 Bad Request)") + .context(ClientErrorExplanation { + code: Some(String::from("SOME_CODE")), + message: Some(String::from("Pass --test-collection-id.")), + }) + .context("Error in create bundle upload endpoint"), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_summary(&result), + [ + "", + "Error Encountered", + "There was an unexpected error that occurred while uploading test results", + "", + "Pass --test-collection-id.", + "", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + "Proceeding with test report...", + "", + "πŸ“š Test Report", + " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn prints_a_limited_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "❕ 9 files found, 5 with issues", + "", + " ❌ test-results/a.xml Has Errors And Warnings", + " β†ͺ ❌ Errors", + " β†ͺ test case name too short", + " β†ͺ test case name too short", + " β†ͺ ⚠️ Warnings", + " β†ͺ report has test cases with missing file or filepath", + " β†ͺ report has test cases with missing timestamp", + " β†ͺ report has test cases with future timestamp", + " β†ͺ report has old (> 24 hour(s)) timestamps", + " β†ͺ report has stale (> 1 hour(s)) timestamps", + " β†ͺ test case or parent has no time duration", + " β†ͺ test case classname too short", + " β†ͺ test case name too long, truncated to 1000", + " ❌ test-results/c.xml Has Errors", + " β†ͺ test case name too short", + " ⚠️ test-results/d.xml Has Warnings", + " β†ͺ test case or parent has no time duration", + "…and 1 more", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_a_full_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + result.validation_report = ValidationReport::Full; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "❕ 9 files found, 5 with issues", + "", + " ❌ test-results/a.xml Has Errors And Warnings", + " β†ͺ ❌ Errors", + " β†ͺ test case name too short", + " β†ͺ test case name too short", + " β†ͺ ⚠️ Warnings", + " β†ͺ report has test cases with missing file or filepath", + " β†ͺ report has test cases with missing timestamp", + " β†ͺ report has test cases with future timestamp", + " β†ͺ report has old (> 24 hour(s)) timestamps", + " β†ͺ report has stale (> 1 hour(s)) timestamps", + " β†ͺ test case or parent has no time duration", + " β†ͺ test case classname too short", + " β†ͺ test case name too long, truncated to 1000", + " β†ͺ test case file or filepath too long", + " ❌ test-results/c.xml Has Errors", + " β†ͺ test case name too short", + " ⚠️ test-results/d.xml Has Warnings", + " β†ͺ test case or parent has no time duration", + " ⚠️ test-results/i.xml Has Warnings", + " β†ͺ report has test cases with missing timestamp", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_a_blank_line_for_no_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + result.validation_report = ValidationReport::None; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_validation_when_every_file_is_correct() { + let mut result = upload_result(5); + result.validations = JunitReportValidations::new(BTreeMap::from([( + String::from("test-results/a.xml"), + validation(Vec::new()), + )])); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "βœ… 1 file found, all fully correct", + "", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn prints_long_failure_messages_when_asked() { + let mut result = upload_result(1); + result.show_failure_messages = true; + result.quarantine_context.exit_code = 1; + let mut message_lines = vec![ + String::from("assertion `left == right` failed\r"), + String::from("\tleft: 1"), + String::new(), + String::from(" "), + String::from("\tright: 2"), + ]; + message_lines.extend((6..=23).map(|n| format!("frame {n}"))); + result.quarantine_context.failures = vec![Test { + failure_message: Some(message_lines.join("\n")), + ..failure("adds", "MathTests", None) + }]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 1 Pass: 0 Fail: 1 Quarantined: 0 Pass Ratio: 0.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " Failure: ", + " assertion `left == right` failed", + " left: 1", + " right: 2", + " frame 6", + " frame 7", + " frame 8", + " frame 9", + " frame 10", + " frame 11", + " frame 12", + " frame 13", + " frame 14", + " frame 15", + " frame 16", + " frame 17", + " frame 18", + " frame 19", + " frame 20", + " …and 3 more lines not shown", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn prints_the_upload_as_a_collection_link() { + let mut result = in_collection(upload_result(2)); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![Test { + id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), + ..failure("adds", "MathTests", None) + }]; + result.guid_scope = Some(TestCaseGuidScope { + test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), + repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), + }); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/tests/81d0929e-fc06-8b61-8506-991d8ebc51ea", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn links_tests_by_short_link_without_collection_ids() { + let mut result = in_collection(upload_result(2)); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn prints_the_bundle_upload_id_when_collection_links_are_hidden() { + let mut result = in_collection(upload_result(2)); + result.hide_test_collection_links = true; + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![Test { + id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), + ..failure("adds", "MathTests", None) + }]; + result.guid_scope = Some(TestCaseGuidScope { + test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), + repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), + }); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/c33a7f64-8f3e-5db9-b37b-2ea870d2441b?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn omits_the_upload_line_without_a_bundle_upload_id() { + let mut result = upload_result(5); + result.meta.base_props.bundle_upload_id = String::new(); + + assert_eq!( + upload_summary(&result), + [ + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); + } + + #[test] + fn groups_failures_by_file_then_suite_then_neither() { + let mut result = upload_result(10); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![ + failure("parses_dates", "DateTests", Some("src/date.rs")), + failure("adds", "MathTests", None), + failure("loads", "", None), + failure("parses_times", "TimeTests", Some("src/date.rs")), + failure("subtracts", "MathTests", None), + failure("boots", "", None), + failure("formats", "", Some("src/a.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 10 Pass: 3 Fail: 7 Quarantined: 0 Pass Ratio: 30.0%", + "", + "❌ 7 tests failed and were not quarantined", + "", + " πŸ“ src/a.rs", + " formats", + " β€· https://app.trunk.io/test-org/flaky-tests/test/formats-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ“ src/date.rs", + " DateTests/parses_dates", + " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_dates-id?repo=trunk-io%2Fanalytics-cli", + " TimeTests/parses_times", + " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_times-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ” Other", + " loads", + " β€· https://app.trunk.io/test-org/flaky-tests/test/loads-id?repo=trunk-io%2Fanalytics-cli", + " boots", + " β€· https://app.trunk.io/test-org/flaky-tests/test/boots-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn shows_three_failures_per_group() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = + ["adds", "subtracts", "multiplies", "divides", "rounds"] + .into_iter() + .map(|name| failure(name, "MathTests", Some("src/math.rs"))) + .collect(); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 0 Fail: 5 Quarantined: 0 Pass Ratio: 0.0%", + "", + "❌ 5 tests failed and were not quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/multiplies", + " β€· https://app.trunk.io/test-org/flaky-tests/test/multiplies-id?repo=trunk-io%2Fanalytics-cli", + " …and 2 more failures in this group", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); + } + + #[test] + fn warns_about_a_non_zero_exit_code_without_failures() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "⚠️ No test failures found, but non zero exit code provided: 1", + "This may indicate that some tests were not run or that there were other issues during the test run.", + ] + ); + } const URL: &str = "https://app.trunk.io/my-org/flaky-tests/test/1"; diff --git a/cli/src/validate_command.rs b/cli/src/validate_command.rs index 0b8ad1d3..27d16322 100644 --- a/cli/src/validate_command.rs +++ b/cli/src/validate_command.rs @@ -959,13 +959,34 @@ impl JunitReportValidations { #[cfg(test)] mod tests { use chrono::DateTime; - use context::junit::junit_path::{ - JunitReportFileWithTestRunnerReport, TestRunnerReport, TestRunnerReportStatus, + use context::junit::{ + junit_path::{ + JunitReportFileWithTestRunnerReport, TestRunnerReport, TestRunnerReportStatus, + }, + parser::{JunitParseIssueInvalid, JunitParseIssueSubOptimal}, }; + use pretty_assertions::assert_eq; use test_utils::inputs::get_test_file_path; use super::*; + fn validate_summary(result: &ValidateRunResult) -> Vec { + result + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() + } + + fn flat_issue(level: JunitValidationLevel, message: &str) -> JunitReportValidationFlatIssue { + JunitReportValidationFlatIssue { + level, + error_message: String::from(message), + ..Default::default() + } + } + #[test] fn test_flatten_glob_returns_all_matches() { let path = get_test_file_path("test_fixtures/junit0*"); @@ -1134,4 +1155,168 @@ mod tests { )]) ); } + + #[test] + fn prints_validation_errors_warnings_and_codeowners() { + let result = ValidateRunResult { + bep_result: None, + file_sets: vec![FileSetResult { + glob: String::from("test-results/**/*.xml"), + file_paths: ["broken", "empty", "multiple", "ok", "unit"] + .map(|name| format!("test-results/{name}.xml")) + .to_vec(), + }], + file_parse_issues: vec![ + ParseIssues { + file_path: String::from("test-results/broken.xml"), + fatal_error: Some(anyhow::anyhow!("unexpected end of file")), + errors: Vec::new(), + warnings: Vec::new(), + }, + ParseIssues { + file_path: String::from("test-results/empty.xml"), + fatal_error: None, + errors: Vec::new(), + warnings: vec![JunitParseIssue::SubOptimal( + JunitParseIssueSubOptimal::ReportNotFound, + )], + }, + ParseIssues { + file_path: String::from("test-results/multiple.xml"), + fatal_error: None, + errors: vec![JunitParseIssue::Invalid( + JunitParseIssueInvalid::ReportMultipleFound, + )], + warnings: Vec::new(), + }, + ], + test_issues: vec![ + TestIssues { + file_path: String::from("test-results/ok.xml"), + num_test_suites: 1, + num_test_cases: 3, + errors: Vec::new(), + warnings: Vec::new(), + }, + TestIssues { + file_path: String::from("test-results/unit.xml"), + num_test_suites: 2, + num_test_cases: 5, + errors: vec![flat_issue( + JunitValidationLevel::Invalid, + "test case name too short", + )], + warnings: vec![ + flat_issue( + JunitValidationLevel::SubOptimal, + "report has test cases with missing file or filepath", + ), + flat_issue( + JunitValidationLevel::SubOptimal, + "test case or parent has no time duration", + ), + ], + }, + ], + codeowners_issues: Some(CodeownersIssues { + file_path: PathBuf::from(".github/CODEOWNERS"), + warnings: vec![String::from( + "CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", + )], + }), + }; + + assert_eq!( + validate_summary(&result), + [ + "πŸ“š Validating 5 files:", + " Files matching: test-results/**/*.xml:", + " test-results/broken.xml", + " test-results/empty.xml", + " test-results/multiple.xml", + " test-results/ok.xml", + " test-results/unit.xml", + "", + "File parse issues", + "test-results/broken.xml", + "1 errors, 0 warnings", + " ❌ Errors:", + " unexpected end of file", + "", + "test-results/empty.xml", + "0 errors, 1 warnings", + " ⚠️ Warnings:", + " no reports found", + "", + "test-results/multiple.xml", + "1 errors, 0 warnings", + " ❌ Errors:", + " multiple reports found", + "", + "Test Validation Issues", + "test-results/ok.xml", + "1 test suites, 3 test cases, 0 errors, 0 warnings", + "test-results/unit.xml", + "2 test suites, 5 test cases, 1 errors, 2 warnings", + " ❌ Errors:", + " test case name too short", + " ⚠️ Warnings:", + " report has test cases with missing file or filepath", + " test case or parent has no time duration", + "", + "Checking for codeowners file...", + " Found codeowners path: \".github/CODEOWNERS\"", + " CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", + "", + "1 valid file, 2 files with warnings, and 3 files with errors, with 5 files total (a file is double counted if it has both errors and warnings)", + "3 warnings, and 3 errors", + ] + ); + } + + #[test] + fn prints_bep_parse_errors_without_a_codeowners_file() { + let result = ValidateRunResult { + bep_result: Some(BepValidateResult { + errors: vec![String::from( + "Error mapping test status for label //app:unit_test: unknown test status 99", + )], + }), + file_sets: vec![FileSetResult { + glob: String::from("bazel-testlogs/app/unit_test/test.xml"), + file_paths: vec![String::from("bazel-testlogs/app/unit_test/test.xml")], + }], + file_parse_issues: Vec::new(), + test_issues: vec![TestIssues { + file_path: String::from("bazel-testlogs/app/unit_test/test.xml"), + num_test_suites: 1, + num_test_cases: 2, + errors: Vec::new(), + warnings: Vec::new(), + }], + codeowners_issues: None, + }; + + assert_eq!( + validate_summary(&result), + [ + "⚠️ BEP file had parse errors: [\"Error mapping test status for label //app:unit_test: unknown test status 99\"]", + "", + "πŸ“š Validating 1 files:", + " Files matching: bazel-testlogs/app/unit_test/test.xml:", + " bazel-testlogs/app/unit_test/test.xml", + "", + "File parse issues", + "Test Validation Issues", + "bazel-testlogs/app/unit_test/test.xml", + "1 test suites, 2 test cases, 0 errors, 0 warnings", + "", + "Checking for codeowners file...", + " No codeowners file found", + "", + "1 valid file, 0 files with warnings, and 0 files with errors, with 1 file total (a file is double counted if it has both errors and warnings)", + "0 warnings, and 0 errors", + ] + ); + } } From 5976f94c53d33c4c7e6670f6b3fb161a9f3aad0d Mon Sep 17 00:00:00 2001 From: Dylan Frankland Date: Wed, 21 Oct 2026 23:05:43 +0000 Subject: [PATCH 2/2] test(cli): move the summary tests into cli/tests The lock-down tests move from src/ into one integration test crate, cli/tests/summary, with a file each for upload, validate and ErrorReport. The existing ErrorReport output test moves with them and now checks every line. report_limiting becomes public so the tests can set --validation-report. The validate tests now run run_validate on fixture files instead of building ValidateRunResult, whose fields are private. Refs TRUNK-22399 Co-Authored-By: Claude Opus 5.5 (1M context) --- cli/src/error_report.rs | 186 ---- cli/src/lib.rs | 2 +- cli/src/upload_command.rs | 978 ++---------------- cli/src/validate_command.rs | 189 +--- .../summary/bep/bep_with_parse_error | 2 + cli/test_fixtures/summary/bep/test.xml | 6 + cli/test_fixtures/summary/validate/CODEOWNERS | 1 + cli/test_fixtures/summary/validate/broken.xml | 4 + cli/test_fixtures/summary/validate/empty.xml | 0 .../summary/validate/multiple.xml | 11 + cli/test_fixtures/summary/validate/unit.xml | 8 + cli/tests/summary/error_report.rs | 183 ++++ cli/tests/summary/main.rs | 12 + cli/tests/summary/upload.rs | 817 +++++++++++++++ cli/tests/summary/validate.rs | 110 ++ 15 files changed, 1242 insertions(+), 1267 deletions(-) create mode 100644 cli/test_fixtures/summary/bep/bep_with_parse_error create mode 100644 cli/test_fixtures/summary/bep/test.xml create mode 100644 cli/test_fixtures/summary/validate/CODEOWNERS create mode 100644 cli/test_fixtures/summary/validate/broken.xml create mode 100644 cli/test_fixtures/summary/validate/empty.xml create mode 100644 cli/test_fixtures/summary/validate/multiple.xml create mode 100644 cli/test_fixtures/summary/validate/unit.xml create mode 100644 cli/tests/summary/error_report.rs create mode 100644 cli/tests/summary/main.rs create mode 100644 cli/tests/summary/upload.rs create mode 100644 cli/tests/summary/validate.rs diff --git a/cli/src/error_report.rs b/cli/src/error_report.rs index 98ea9b20..c13284f2 100644 --- a/cli/src/error_report.rs +++ b/cli/src/error_report.rs @@ -240,32 +240,6 @@ impl EndOutput for ErrorReport { } } -#[cfg(test)] -fn error_summary(report: &ErrorReport) -> Vec { - report - .output() - .unwrap() - .iter() - .map(Line::to_unstyled) - .collect() -} - -#[cfg(test)] -pub(crate) fn unauthorized_error() -> anyhow::Error { - let response = http::Response::builder().status(401).body("").unwrap(); - anyhow::Error::from( - reqwest::Response::from(response) - .error_for_status() - .unwrap_err(), - ) -} - -#[cfg(test)] -fn connection_refused_error() -> anyhow::Error { - anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)) - .context("Failed to create bundle upload") -} - #[test] fn adds_settings_if_domain_present() { let host = "https://api.fake-trunk.io"; @@ -275,163 +249,3 @@ fn adds_settings_if_domain_present() { "Hint: You can find it under the settings page at https://app.fake-trunk.io/fake-org-slug/settings" ) } - -#[test] -fn prints_a_client_error_explanation_beneath_the_base_message() { - let error = anyhow::anyhow!("HTTP status client error (400 Bad Request)") - .context(ClientErrorExplanation { - code: Some(String::from("SOME_CODE")), - message: Some(String::from("Pass --test-collection-id.")), - }) - .context("Error in create bundle upload endpoint"); - let report = ErrorReport::new( - error, - String::from("fake-org-slug"), - Some(String::from("There was an unexpected error")), - ); - - let text = error_summary(&report); - - assert_eq!( - text[..4], - [ - "Error Encountered", - "There was an unexpected error", - "", - "Pass --test-collection-id.", - ] - ); -} - -#[test] -fn prints_connection_refused_beneath_the_base_message() { - let report = ErrorReport::new( - connection_refused_error(), - String::from("fake-org-slug"), - Some(String::from("There was an unexpected error")), - ); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "There was an unexpected error", - "", - "Unable to connect to trunk's server", - ] - ); -} - -#[test] -fn prints_connection_refused_without_a_base_message() { - let report = ErrorReport::new( - connection_refused_error(), - String::from("fake-org-slug"), - None, - ); - - assert_eq!( - error_summary(&report), - ["Error Encountered", "Unable to connect to trunk's server"] - ); -} - -#[test] -fn prints_unauthorized_with_a_settings_hint() { - let report = ErrorReport::new(unauthorized_error(), String::from("fake-org-slug"), None); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "", - "", - "Unathorized access, your Trunk organization URL slug or token may be incorrect", - " Hint: You can find it under the settings page at https://app.trunk.io/fake-org-slug/settings", - ] - ); -} - -#[test] -fn prints_a_generic_error_without_a_base_message() { - let report = ErrorReport::new( - anyhow::anyhow!("--allow-forked-pr-uploads requires --test-collection-id"), - String::from("fake-org-slug"), - None, - ); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "--allow-forked-pr-uploads requires --test-collection-id", - "For more help, contact us at https://slack.trunk.io/", - "", - "Errors occurred during execution, using quarantining exit code", - ] - ); -} - -#[test] -fn prints_an_interrupting_error_with_its_exit_code() { - let report = ErrorReport::new( - anyhow::anyhow!(InterruptingError::new( - "No test output files found to validate" - )), - String::from("fake-org-slug"), - None, - ); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "No test output files found to validate", - "For more help, contact us at https://slack.trunk.io/", - "", - "Errors occurred during execution, returning exit code: 70", - ] - ); -} - -#[test] -fn prints_the_default_exit_code() { - let mut report = ErrorReport::new( - anyhow::anyhow!("something went wrong"), - String::from("fake-org-slug"), - None, - ); - report.context.exit_code = Some(exitcode::OK); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "something went wrong", - "For more help, contact us at https://slack.trunk.io/", - "", - "No errors occurred, returning default exit code: 0", - ] - ); -} - -#[test] -fn prints_any_other_exit_code() { - let mut report = ErrorReport::new( - anyhow::anyhow!("something went wrong"), - String::from("fake-org-slug"), - None, - ); - report.context.exit_code = Some(exitcode::USAGE); - - assert_eq!( - error_summary(&report), - [ - "Error Encountered", - "something went wrong", - "For more help, contact us at https://slack.trunk.io/", - "", - "Errors occurred during execution, returning exit code: 64", - ] - ); -} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 0c7bd3ba..92ce0bd4 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -2,7 +2,7 @@ pub mod context; pub mod context_quarantine; pub mod error_report; pub mod print; -mod report_limiting; +pub mod report_limiting; pub mod test_command; pub mod upload_command; pub mod validate_command; diff --git a/cli/src/upload_command.rs b/cli/src/upload_command.rs index 9e35fc2b..34425fc4 100644 --- a/cli/src/upload_command.rs +++ b/cli/src/upload_command.rs @@ -1080,94 +1080,96 @@ impl EndOutput for UploadRunResult { } } // Helper to render a group - let mut render_group = - |header: &str, color: Color, group: &Vec<&Test>| -> anyhow::Result<()> { - output.push(Line::from_iter([Span::new_styled( - style(header.to_string()) - .with(color) - .attribute(Attribute::Bold), - )?])); - for test in group.iter().take(3) { - let output_name = format!( - "{}{}{}", - test.parent_name, - if test.parent_name.is_empty() { "" } else { "/" }, - test.name - ); - let mut test_line = Line::from_iter([Span::new_styled_lossy( - style(output_name.to_string()).attribute(Attribute::Bold), - )]); - test_line.pad_left(2); - output.push(test_line); - let link = url_for_test_case( - &self.api_address, - &self.quarantine_context.org_url_slug, - &self.quarantine_context.repo, - test, - test_collection_short_id, - guid_scope, - )?; - let mut link_output = Line::from_iter([ - Span::new_unstyled("β€· ")?, - hyperlink_span(link, Attribute::Underlined)?, - ]); - link_output.pad_left(4); - output.push(link_output); - // Display failure message if present and enabled - // TODO: show_failure_messages is a temporary flag to show failure messages - // in the output. It should be removed once we are confident in this flow - // and we should use the validation report flag instead. - if self.show_failure_messages && test.failure_message.is_some() { - let failure_message = test.failure_message.as_ref().unwrap(); - let lines: Vec<&str> = failure_message.split('\n').collect(); - let max_lines = 20; - let shown_lines = lines.iter().take(max_lines); - let mut failure_header = Line::from_iter([Span::new_styled( - style("Failure: ".to_string()).with(Color::DarkGrey), - )?]); - failure_header.pad_left(4); - output.push(failure_header); - for (j, line) in shown_lines.enumerate() { - let sanitized_line = line.replace('\t', " ").replace('\r', ""); - if !sanitized_line.trim().is_empty() || j == 0 { - let mut failure_output = Line::from_iter([ - Span::new_unstyled(" ")?, - Span::new_styled_lossy( - style(sanitized_line.to_string()) - .with(Color::Grey) - .attribute(Attribute::Italic), - ), - ]); - failure_output.pad_left(4); - output.push(failure_output); - } - } - if lines.len() > max_lines { - let omitted = lines.len() - max_lines; - let mut more_output = Line::from_iter([ + let mut render_group = |header: &str, + color: Color, + group: &Vec<&Test>| + -> anyhow::Result<()> { + output.push(Line::from_iter([Span::new_styled( + style(header.to_string()) + .with(color) + .attribute(Attribute::Bold), + )?])); + for test in group.iter().take(3) { + let output_name = format!( + "{}{}{}", + test.parent_name, + if test.parent_name.is_empty() { "" } else { "/" }, + test.name + ); + let mut test_line = Line::from_iter([Span::new_styled_lossy( + style(output_name.to_string()).attribute(Attribute::Bold), + )]); + test_line.pad_left(2); + output.push(test_line); + let link = url_for_test_case( + &self.api_address, + &self.quarantine_context.org_url_slug, + &self.quarantine_context.repo, + test, + test_collection_short_id, + guid_scope, + )?; + let mut link_output = Line::from_iter([ + Span::new_unstyled("β€· ")?, + hyperlink_span(link, Attribute::Underlined)?, + ]); + link_output.pad_left(4); + output.push(link_output); + // Display failure message if present and enabled + // TODO: show_failure_messages is a temporary flag to show failure messages + // in the output. It should be removed once we are confident in this flow + // and we should use the validation report flag instead. + if self.show_failure_messages && test.failure_message.is_some() { + let failure_message = test.failure_message.as_ref().unwrap(); + let lines: Vec<&str> = failure_message.split('\n').collect(); + let max_lines = 20; + let shown_lines = lines.iter().take(max_lines); + let mut failure_header = Line::from_iter([Span::new_styled( + style("Failure: ".to_string()).with(Color::DarkGrey), + )?]); + failure_header.pad_left(4); + output.push(failure_header); + for (j, line) in shown_lines.enumerate() { + let sanitized_line = line.replace('\t', " ").replace('\r', ""); + if !sanitized_line.trim().is_empty() || j == 0 { + let mut failure_output = Line::from_iter([ Span::new_unstyled(" ")?, - Span::new_styled( - style(format!("…and {omitted} more lines not shown")) + Span::new_styled_lossy( + style(sanitized_line.to_string()) .with(Color::Grey) .attribute(Attribute::Italic), - )?, + ), ]); - more_output.pad_left(4); - output.push(more_output); + failure_output.pad_left(4); + output.push(failure_output); } } + if lines.len() > max_lines { + let omitted = lines.len() - max_lines; + let mut more_output = Line::from_iter([ + Span::new_unstyled(" ")?, + Span::new_styled( + style(format!("…and {omitted} more lines not shown")) + .with(Color::Grey) + .attribute(Attribute::Italic), + )?, + ]); + more_output.pad_left(4); + output.push(more_output); + } } - if group.len() > 3 { - let mut more_failures = Line::from_iter([Span::new_unstyled(format!( - "…and {} more failures in this group", - group.len() - 3 - ))?]); - more_failures.pad_left(2); - output.push(more_failures); - } - output.push(Line::default()); - Ok(()) - }; + } + if group.len() > 3 { + let mut more_failures = Line::from_iter([Span::new_unstyled(format!( + "…and {} more failures in this group", + group.len() - 3 + ))?]); + more_failures.pad_left(2); + output.push(more_failures); + } + output.push(Line::default()); + Ok(()) + }; // Render file path groups (cyan) for (file, group) in &groups { render_group(&format!("πŸ“ {}", file), Color::Cyan, group)?; @@ -1288,819 +1290,9 @@ impl EndOutput for UploadRunResult { #[cfg(test)] mod tests { - use std::collections::{BTreeMap, HashMap}; - - use api::{client::ClientErrorExplanation, urls::TestCaseGuidScope}; - use bundle::{ - BundleMetaBaseProps, BundleMetaDebugProps, BundleMetaJunitProps, TestCollectionProps, - }; - use context::{ - junit::validator::{ - JunitReportValidation, JunitReportValidationIssueSubOptimal, - JunitTestCaseValidationIssueInvalid, JunitTestCaseValidationIssueSubOptimal, - JunitValidationIssue, JunitValidationIssueType, - }, - repo::{BundleRepo, RepoUrlParts}, - }; - use pretty_assertions::assert_eq; - - use super::*; - use crate::{context_quarantine::QuarantineFetchStatus, error_report::unauthorized_error}; - - const BUNDLE_UPLOAD_ID: &str = "4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d"; - const UPLOAD_ERROR_MESSAGE: &str = - "There was an unexpected error that occurred while uploading test results"; - - fn upload_summary(result: &UploadRunResult) -> Vec { - result - .output() - .unwrap() - .iter() - .map(Line::to_unstyled) - .collect() - } - - fn repo() -> RepoUrlParts { - RepoUrlParts { - host: String::from("github.com"), - owner: String::from("trunk-io"), - name: String::from("analytics-cli"), - } - } - - fn upload_result(num_tests: usize) -> UploadRunResult { - let mut quarantine_context = QuarantineContext::skip_fetch(Vec::new()); - quarantine_context.repo = repo(); - quarantine_context.org_url_slug = String::from("test-org"); - UploadRunResult { - error_report: None, - quarantine_context, - meta: BundleMeta { - base_props: BundleMetaBaseProps { - version: String::new(), - cli_version: String::new(), - org: String::from("test-org"), - test_collection: None, - repo: BundleRepo { - repo: repo(), - ..Default::default() - }, - bundle_upload_id: String::from(BUNDLE_UPLOAD_ID), - tags: Vec::new(), - file_sets: Vec::new(), - envs: HashMap::new(), - upload_time_epoch: 0, - test_command: None, - os_info: None, - quarantined_tests: Vec::new(), - codeowners: None, - use_uncloned_repo: None, - quarantine_resolution_mode: QuarantineResolutionMode::default(), - }, - junit_props: BundleMetaJunitProps { - num_files: 1, - num_tests, - }, - debug_props: BundleMetaDebugProps { - command_line: String::new(), - trunk_envs: HashMap::new(), - }, - bundle_upload_id_v2: String::new(), - variant: None, - internal_bundled_file: None, - failed_tests: Vec::new(), - }, - validations: JunitReportValidations::new(BTreeMap::new()), - validation_report: ValidationReport::Limited, - show_failure_messages: false, - test_collection_short_id: None, - hide_test_collection_links: false, - api_address: String::from("https://api.trunk.io"), - guid_scope: None, - } - } - - fn in_collection(mut result: UploadRunResult) -> UploadRunResult { - result.test_collection_short_id = Some(String::from("abc123")); - result.meta.base_props.test_collection = Some(TestCollectionProps { - short_id: String::from("abc123"), - bundle_meta_id: String::from("0b6f4c1e-8a7d-4e2b-9c3f-5d1a2b3c4d5e"), - bundle_meta_created_at: String::from("2026-10-21T12:00:00Z"), - }); - result - } - - fn failure(name: &str, parent_name: &str, file: Option<&str>) -> Test { - Test { - name: String::from(name), - parent_name: String::from(parent_name), - class_name: None, - file: file.map(String::from), - id: format!("{name}-id"), - timestamp_millis: None, - is_quarantined: false, - failure_message: None, - variant: None, - } - } - - fn warning(issue: JunitTestCaseValidationIssueSubOptimal) -> JunitValidationIssueType { - JunitValidationIssueType::TestCase(JunitValidationIssue::SubOptimal(issue)) - } - - fn report_warning(issue: JunitReportValidationIssueSubOptimal) -> JunitValidationIssueType { - JunitValidationIssueType::Report(JunitValidationIssue::SubOptimal(issue)) - } - - fn name_too_short() -> JunitValidationIssueType { - JunitValidationIssueType::TestCase(JunitValidationIssue::Invalid( - JunitTestCaseValidationIssueInvalid::TestCaseNameTooShort(String::new()), - )) - } - - fn validation( - all_issues: Vec, - ) -> anyhow::Result { - Ok(JunitReportValidation { - all_issues, - ..Default::default() - }) - } - - fn validations() -> JunitReportValidations { - JunitReportValidations::new(BTreeMap::from([ - ( - String::from("test-results/a.xml"), - validation(vec![ - name_too_short(), - name_too_short(), - report_warning( - JunitReportValidationIssueSubOptimal::TestCasesFileOrFilepathMissing, - ), - report_warning(JunitReportValidationIssueSubOptimal::MissingTimestamps), - report_warning(JunitReportValidationIssueSubOptimal::FutureTimestamps), - report_warning(JunitReportValidationIssueSubOptimal::OldTimestamps), - report_warning(JunitReportValidationIssueSubOptimal::StaleTimestamps), - warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration), - warning( - JunitTestCaseValidationIssueSubOptimal::TestCaseClassnameTooShort( - String::new(), - ), - ), - warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNameTooLong( - String::new(), - )), - warning( - JunitTestCaseValidationIssueSubOptimal::TestCaseFileOrFilepathTooLong( - String::new(), - ), - ), - ]), - ), - ( - String::from("test-results/b.xml"), - Err(anyhow::anyhow!("unexpected end of file")), - ), - ( - String::from("test-results/c.xml"), - validation(vec![name_too_short()]), - ), - ( - String::from("test-results/d.xml"), - validation(vec![warning( - JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration, - )]), - ), - (String::from("test-results/e.xml"), validation(Vec::new())), - (String::from("test-results/f.xml"), validation(Vec::new())), - (String::from("test-results/g.xml"), validation(Vec::new())), - (String::from("test-results/h.xml"), validation(Vec::new())), - ( - String::from("test-results/i.xml"), - validation(vec![report_warning( - JunitReportValidationIssueSubOptimal::MissingTimestamps, - )]), - ), - ])) - } - - #[test] - fn prints_a_passing_run() { - let result = upload_result(5); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_that_no_tests_were_found() { - let result = upload_result(0); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 0 Pass: 0 Fail: 0 Quarantined: 0 Pass Ratio: N/A", - "", - "⚠️ No tests were found in the provided test results", - ] - ); - } + use superconsole::{Line, style::Attribute}; - #[test] - fn prints_failures_that_were_all_quarantined() { - let mut result = upload_result(4); - result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; - result - .quarantine_context - .quarantine_status - .group_is_quarantined = true; - result - .quarantine_context - .quarantine_status - .quarantine_results = vec![ - failure("adds", "MathTests", Some("src/math.rs")), - failure("subtracts", "MathTests", Some("src/math.rs")), - ]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 4 Pass: 2 Fail: 0 Quarantined: 2 Pass Ratio: 50.0%", - "", - "β€οΈβ€πŸ©Ή 2 tests failed and were quarantined", - "", - " πŸ“ src/math.rs", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/subtracts", - " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success (0)", - ] - ); - } - - #[test] - fn prints_failures_that_were_not_quarantined() { - let mut result = upload_result(5); - result.quarantine_context.exit_code = 1; - result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; - result - .quarantine_context - .quarantine_status - .quarantine_results = vec![failure("adds", "MathTests", Some("src/math.rs"))]; - result.quarantine_context.failures = vec![ - failure("subtracts", "MathTests", Some("src/math.rs")), - failure("divides", "MathTests", Some("src/math.rs")), - ]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 2 Fail: 2 Quarantined: 1 Pass Ratio: 40.0%", - "", - "β€οΈβ€πŸ©Ή 1 test failed and was quarantined", - "", - " πŸ“ src/math.rs", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "❌ 2 tests failed and were not quarantined", - "", - " πŸ“ src/math.rs", - " MathTests/subtracts", - " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/divides", - " β€· https://app.trunk.io/test-org/flaky-tests/test/divides-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn prints_a_warning_when_the_quarantine_lookup_failed() { - let mut result = upload_result(3); - result.quarantine_context.exit_code = 1; - result.quarantine_context.fetch_status = - QuarantineFetchStatus::FetchFailed(anyhow::anyhow!("request timed out")); - result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", - " We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn stops_at_an_error_that_blocks_quarantining() { - let mut result = upload_result(3); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; - result.error_report = Some(ErrorReport::new( - unauthorized_error(), - String::from("test-org"), - Some(String::from(UPLOAD_ERROR_MESSAGE)), - )); - - assert_eq!( - upload_summary(&result), - [ - "", - "Error Encountered", - "There was an unexpected error that occurred while uploading test results", - "", - "Unathorized access, your Trunk organization URL slug or token may be incorrect", - " Hint: You can find it under the settings page at https://app.trunk.io/test-org/settings", - ] - ); - } - - #[test] - fn proceeds_past_an_upload_error() { - let mut result = upload_result(3); - result.meta.base_props.bundle_upload_id = String::new(); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; - result.error_report = Some(ErrorReport::new( - anyhow::anyhow!("HTTP status client error (400 Bad Request)") - .context(ClientErrorExplanation { - code: Some(String::from("SOME_CODE")), - message: Some(String::from("Pass --test-collection-id.")), - }) - .context("Error in create bundle upload endpoint"), - String::from("test-org"), - Some(String::from(UPLOAD_ERROR_MESSAGE)), - )); - - assert_eq!( - upload_summary(&result), - [ - "", - "Error Encountered", - "There was an unexpected error that occurred while uploading test results", - "", - "Pass --test-collection-id.", - "", - "For more help, contact us at https://slack.trunk.io/", - "", - "Errors occurred during execution, using quarantining exit code", - "Proceeding with test report...", - "", - "πŸ“š Test Report", - " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn prints_a_limited_validation_report() { - let mut result = upload_result(5); - result.validations = validations(); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“‚ File Validation", - "", - "❕ 9 files found, 5 with issues", - "", - " ❌ test-results/a.xml Has Errors And Warnings", - " β†ͺ ❌ Errors", - " β†ͺ test case name too short", - " β†ͺ test case name too short", - " β†ͺ ⚠️ Warnings", - " β†ͺ report has test cases with missing file or filepath", - " β†ͺ report has test cases with missing timestamp", - " β†ͺ report has test cases with future timestamp", - " β†ͺ report has old (> 24 hour(s)) timestamps", - " β†ͺ report has stale (> 1 hour(s)) timestamps", - " β†ͺ test case or parent has no time duration", - " β†ͺ test case classname too short", - " β†ͺ test case name too long, truncated to 1000", - " ❌ test-results/c.xml Has Errors", - " β†ͺ test case name too short", - " ⚠️ test-results/d.xml Has Warnings", - " β†ͺ test case or parent has no time duration", - "…and 1 more", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_a_full_validation_report() { - let mut result = upload_result(5); - result.validations = validations(); - result.validation_report = ValidationReport::Full; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“‚ File Validation", - "", - "❕ 9 files found, 5 with issues", - "", - " ❌ test-results/a.xml Has Errors And Warnings", - " β†ͺ ❌ Errors", - " β†ͺ test case name too short", - " β†ͺ test case name too short", - " β†ͺ ⚠️ Warnings", - " β†ͺ report has test cases with missing file or filepath", - " β†ͺ report has test cases with missing timestamp", - " β†ͺ report has test cases with future timestamp", - " β†ͺ report has old (> 24 hour(s)) timestamps", - " β†ͺ report has stale (> 1 hour(s)) timestamps", - " β†ͺ test case or parent has no time duration", - " β†ͺ test case classname too short", - " β†ͺ test case name too long, truncated to 1000", - " β†ͺ test case file or filepath too long", - " ❌ test-results/c.xml Has Errors", - " β†ͺ test case name too short", - " ⚠️ test-results/d.xml Has Warnings", - " β†ͺ test case or parent has no time duration", - " ⚠️ test-results/i.xml Has Warnings", - " β†ͺ report has test cases with missing timestamp", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_a_blank_line_for_no_validation_report() { - let mut result = upload_result(5); - result.validations = validations(); - result.validation_report = ValidationReport::None; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_validation_when_every_file_is_correct() { - let mut result = upload_result(5); - result.validations = JunitReportValidations::new(BTreeMap::from([( - String::from("test-results/a.xml"), - validation(Vec::new()), - )])); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“‚ File Validation", - "", - "βœ… 1 file found, all fully correct", - "", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn prints_long_failure_messages_when_asked() { - let mut result = upload_result(1); - result.show_failure_messages = true; - result.quarantine_context.exit_code = 1; - let mut message_lines = vec![ - String::from("assertion `left == right` failed\r"), - String::from("\tleft: 1"), - String::new(), - String::from(" "), - String::from("\tright: 2"), - ]; - message_lines.extend((6..=23).map(|n| format!("frame {n}"))); - result.quarantine_context.failures = vec![Test { - failure_message: Some(message_lines.join("\n")), - ..failure("adds", "MathTests", None) - }]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 1 Pass: 0 Fail: 1 Quarantined: 0 Pass Ratio: 0.0%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " Failure: ", - " assertion `left == right` failed", - " left: 1", - " right: 2", - " frame 6", - " frame 7", - " frame 8", - " frame 9", - " frame 10", - " frame 11", - " frame 12", - " frame 13", - " frame 14", - " frame 15", - " frame 16", - " frame 17", - " frame 18", - " frame 19", - " frame 20", - " …and 3 more lines not shown", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn prints_the_upload_as_a_collection_link() { - let mut result = in_collection(upload_result(2)); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![Test { - id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), - ..failure("adds", "MathTests", None) - }]; - result.guid_scope = Some(TestCaseGuidScope { - test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), - repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), - }); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", - "", - "πŸ“š Test Report", - " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/tests/81d0929e-fc06-8b61-8506-991d8ebc51ea", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn links_tests_by_short_link_without_collection_ids() { - let mut result = in_collection(upload_result(2)); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", - "", - "πŸ“š Test Report", - " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/adds-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn prints_the_bundle_upload_id_when_collection_links_are_hidden() { - let mut result = in_collection(upload_result(2)); - result.hide_test_collection_links = true; - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![Test { - id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), - ..failure("adds", "MathTests", None) - }]; - result.guid_scope = Some(TestCaseGuidScope { - test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), - repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), - }); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", - "", - "❌ 1 test failed and was not quarantined", - "", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/c33a7f64-8f3e-5db9-b37b-2ea870d2441b?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn omits_the_upload_line_without_a_bundle_upload_id() { - let mut result = upload_result(5); - result.meta.base_props.bundle_upload_id = String::new(); - - assert_eq!( - upload_summary(&result), - [ - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "πŸŽ‰ No test failures found!", - ] - ); - } - - #[test] - fn groups_failures_by_file_then_suite_then_neither() { - let mut result = upload_result(10); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = vec![ - failure("parses_dates", "DateTests", Some("src/date.rs")), - failure("adds", "MathTests", None), - failure("loads", "", None), - failure("parses_times", "TimeTests", Some("src/date.rs")), - failure("subtracts", "MathTests", None), - failure("boots", "", None), - failure("formats", "", Some("src/a.rs")), - ]; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 10 Pass: 3 Fail: 7 Quarantined: 0 Pass Ratio: 30.0%", - "", - "❌ 7 tests failed and were not quarantined", - "", - " πŸ“ src/a.rs", - " formats", - " β€· https://app.trunk.io/test-org/flaky-tests/test/formats-id?repo=trunk-io%2Fanalytics-cli", - " ", - " πŸ“ src/date.rs", - " DateTests/parses_dates", - " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_dates-id?repo=trunk-io%2Fanalytics-cli", - " TimeTests/parses_times", - " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_times-id?repo=trunk-io%2Fanalytics-cli", - " ", - " πŸ“¦ MathTests", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/subtracts", - " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", - " ", - " πŸ” Other", - " loads", - " β€· https://app.trunk.io/test-org/flaky-tests/test/loads-id?repo=trunk-io%2Fanalytics-cli", - " boots", - " β€· https://app.trunk.io/test-org/flaky-tests/test/boots-id?repo=trunk-io%2Fanalytics-cli", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn shows_three_failures_per_group() { - let mut result = upload_result(5); - result.quarantine_context.exit_code = 1; - result.quarantine_context.failures = - ["adds", "subtracts", "multiplies", "divides", "rounds"] - .into_iter() - .map(|name| failure(name, "MathTests", Some("src/math.rs"))) - .collect(); - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 0 Fail: 5 Quarantined: 0 Pass Ratio: 0.0%", - "", - "❌ 5 tests failed and were not quarantined", - "", - " πŸ“ src/math.rs", - " MathTests/adds", - " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/subtracts", - " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", - " MathTests/multiplies", - " β€· https://app.trunk.io/test-org/flaky-tests/test/multiplies-id?repo=trunk-io%2Fanalytics-cli", - " …and 2 more failures in this group", - " ", - "", - "⚠️ Some test failures were not quarantined, using exit code: 1", - ] - ); - } - - #[test] - fn warns_about_a_non_zero_exit_code_without_failures() { - let mut result = upload_result(5); - result.quarantine_context.exit_code = 1; - - assert_eq!( - upload_summary(&result), - [ - "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", - "", - "πŸ“š Test Report", - " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", - "", - "⚠️ No test failures found, but non zero exit code provided: 1", - "This may indicate that some tests were not run or that there were other issues during the test run.", - ] - ); - } + use super::url_span; const URL: &str = "https://app.trunk.io/my-org/flaky-tests/test/1"; diff --git a/cli/src/validate_command.rs b/cli/src/validate_command.rs index 27d16322..0b8ad1d3 100644 --- a/cli/src/validate_command.rs +++ b/cli/src/validate_command.rs @@ -959,34 +959,13 @@ impl JunitReportValidations { #[cfg(test)] mod tests { use chrono::DateTime; - use context::junit::{ - junit_path::{ - JunitReportFileWithTestRunnerReport, TestRunnerReport, TestRunnerReportStatus, - }, - parser::{JunitParseIssueInvalid, JunitParseIssueSubOptimal}, + use context::junit::junit_path::{ + JunitReportFileWithTestRunnerReport, TestRunnerReport, TestRunnerReportStatus, }; - use pretty_assertions::assert_eq; use test_utils::inputs::get_test_file_path; use super::*; - fn validate_summary(result: &ValidateRunResult) -> Vec { - result - .output() - .unwrap() - .iter() - .map(Line::to_unstyled) - .collect() - } - - fn flat_issue(level: JunitValidationLevel, message: &str) -> JunitReportValidationFlatIssue { - JunitReportValidationFlatIssue { - level, - error_message: String::from(message), - ..Default::default() - } - } - #[test] fn test_flatten_glob_returns_all_matches() { let path = get_test_file_path("test_fixtures/junit0*"); @@ -1155,168 +1134,4 @@ mod tests { )]) ); } - - #[test] - fn prints_validation_errors_warnings_and_codeowners() { - let result = ValidateRunResult { - bep_result: None, - file_sets: vec![FileSetResult { - glob: String::from("test-results/**/*.xml"), - file_paths: ["broken", "empty", "multiple", "ok", "unit"] - .map(|name| format!("test-results/{name}.xml")) - .to_vec(), - }], - file_parse_issues: vec![ - ParseIssues { - file_path: String::from("test-results/broken.xml"), - fatal_error: Some(anyhow::anyhow!("unexpected end of file")), - errors: Vec::new(), - warnings: Vec::new(), - }, - ParseIssues { - file_path: String::from("test-results/empty.xml"), - fatal_error: None, - errors: Vec::new(), - warnings: vec![JunitParseIssue::SubOptimal( - JunitParseIssueSubOptimal::ReportNotFound, - )], - }, - ParseIssues { - file_path: String::from("test-results/multiple.xml"), - fatal_error: None, - errors: vec![JunitParseIssue::Invalid( - JunitParseIssueInvalid::ReportMultipleFound, - )], - warnings: Vec::new(), - }, - ], - test_issues: vec![ - TestIssues { - file_path: String::from("test-results/ok.xml"), - num_test_suites: 1, - num_test_cases: 3, - errors: Vec::new(), - warnings: Vec::new(), - }, - TestIssues { - file_path: String::from("test-results/unit.xml"), - num_test_suites: 2, - num_test_cases: 5, - errors: vec![flat_issue( - JunitValidationLevel::Invalid, - "test case name too short", - )], - warnings: vec![ - flat_issue( - JunitValidationLevel::SubOptimal, - "report has test cases with missing file or filepath", - ), - flat_issue( - JunitValidationLevel::SubOptimal, - "test case or parent has no time duration", - ), - ], - }, - ], - codeowners_issues: Some(CodeownersIssues { - file_path: PathBuf::from(".github/CODEOWNERS"), - warnings: vec![String::from( - "CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", - )], - }), - }; - - assert_eq!( - validate_summary(&result), - [ - "πŸ“š Validating 5 files:", - " Files matching: test-results/**/*.xml:", - " test-results/broken.xml", - " test-results/empty.xml", - " test-results/multiple.xml", - " test-results/ok.xml", - " test-results/unit.xml", - "", - "File parse issues", - "test-results/broken.xml", - "1 errors, 0 warnings", - " ❌ Errors:", - " unexpected end of file", - "", - "test-results/empty.xml", - "0 errors, 1 warnings", - " ⚠️ Warnings:", - " no reports found", - "", - "test-results/multiple.xml", - "1 errors, 0 warnings", - " ❌ Errors:", - " multiple reports found", - "", - "Test Validation Issues", - "test-results/ok.xml", - "1 test suites, 3 test cases, 0 errors, 0 warnings", - "test-results/unit.xml", - "2 test suites, 5 test cases, 1 errors, 2 warnings", - " ❌ Errors:", - " test case name too short", - " ⚠️ Warnings:", - " report has test cases with missing file or filepath", - " test case or parent has no time duration", - "", - "Checking for codeowners file...", - " Found codeowners path: \".github/CODEOWNERS\"", - " CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", - "", - "1 valid file, 2 files with warnings, and 3 files with errors, with 5 files total (a file is double counted if it has both errors and warnings)", - "3 warnings, and 3 errors", - ] - ); - } - - #[test] - fn prints_bep_parse_errors_without_a_codeowners_file() { - let result = ValidateRunResult { - bep_result: Some(BepValidateResult { - errors: vec![String::from( - "Error mapping test status for label //app:unit_test: unknown test status 99", - )], - }), - file_sets: vec![FileSetResult { - glob: String::from("bazel-testlogs/app/unit_test/test.xml"), - file_paths: vec![String::from("bazel-testlogs/app/unit_test/test.xml")], - }], - file_parse_issues: Vec::new(), - test_issues: vec![TestIssues { - file_path: String::from("bazel-testlogs/app/unit_test/test.xml"), - num_test_suites: 1, - num_test_cases: 2, - errors: Vec::new(), - warnings: Vec::new(), - }], - codeowners_issues: None, - }; - - assert_eq!( - validate_summary(&result), - [ - "⚠️ BEP file had parse errors: [\"Error mapping test status for label //app:unit_test: unknown test status 99\"]", - "", - "πŸ“š Validating 1 files:", - " Files matching: bazel-testlogs/app/unit_test/test.xml:", - " bazel-testlogs/app/unit_test/test.xml", - "", - "File parse issues", - "Test Validation Issues", - "bazel-testlogs/app/unit_test/test.xml", - "1 test suites, 2 test cases, 0 errors, 0 warnings", - "", - "Checking for codeowners file...", - " No codeowners file found", - "", - "1 valid file, 0 files with warnings, and 0 files with errors, with 1 file total (a file is double counted if it has both errors and warnings)", - "0 warnings, and 0 errors", - ] - ); - } } diff --git a/cli/test_fixtures/summary/bep/bep_with_parse_error b/cli/test_fixtures/summary/bep/bep_with_parse_error new file mode 100644 index 00000000..f50844ef --- /dev/null +++ b/cli/test_fixtures/summary/bep/bep_with_parse_error @@ -0,0 +1,2 @@ +{"id":{"testResult":{"label":"//app:unit_test","run":1,"shard":1,"attempt":1}},"testResult":{"testActionOutput":[{"name":"test.xml","uri":"file://test_fixtures/summary/bep/test.xml"}],"status":"PASSED"}} +{"id":{"testResult": diff --git a/cli/test_fixtures/summary/bep/test.xml b/cli/test_fixtures/summary/bep/test.xml new file mode 100644 index 00000000..f9f77a91 --- /dev/null +++ b/cli/test_fixtures/summary/bep/test.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/cli/test_fixtures/summary/validate/CODEOWNERS b/cli/test_fixtures/summary/validate/CODEOWNERS new file mode 100644 index 00000000..b1ece579 --- /dev/null +++ b/cli/test_fixtures/summary/validate/CODEOWNERS @@ -0,0 +1 @@ +* @example-org/example-team diff --git a/cli/test_fixtures/summary/validate/broken.xml b/cli/test_fixtures/summary/validate/broken.xml new file mode 100644 index 00000000..a855a14f --- /dev/null +++ b/cli/test_fixtures/summary/validate/broken.xml @@ -0,0 +1,4 @@ + + + + diff --git a/cli/test_fixtures/summary/validate/empty.xml b/cli/test_fixtures/summary/validate/empty.xml new file mode 100644 index 00000000..e69de29b diff --git a/cli/test_fixtures/summary/validate/multiple.xml b/cli/test_fixtures/summary/validate/multiple.xml new file mode 100644 index 00000000..314d88a3 --- /dev/null +++ b/cli/test_fixtures/summary/validate/multiple.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/cli/test_fixtures/summary/validate/unit.xml b/cli/test_fixtures/summary/validate/unit.xml new file mode 100644 index 00000000..76812c04 --- /dev/null +++ b/cli/test_fixtures/summary/validate/unit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/cli/tests/summary/error_report.rs b/cli/tests/summary/error_report.rs new file mode 100644 index 00000000..d946938b --- /dev/null +++ b/cli/tests/summary/error_report.rs @@ -0,0 +1,183 @@ +use api::client::ClientErrorExplanation; +use display::end_output::EndOutput; +use pretty_assertions::assert_eq; +use superconsole::Line; +use trunk_analytics_cli::error_report::{ErrorReport, InterruptingError}; + +use crate::unauthorized_error; + +fn error_summary(report: &ErrorReport) -> Vec { + report + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() +} + +fn connection_refused_error() -> anyhow::Error { + anyhow::Error::from(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)) + .context("Failed to create bundle upload") +} + +#[test] +fn prints_a_client_error_explanation_beneath_the_base_message() { + let error = anyhow::anyhow!("HTTP status client error (400 Bad Request)") + .context(ClientErrorExplanation { + code: Some(String::from("SOME_CODE")), + message: Some(String::from("Pass --test-collection-id.")), + }) + .context("Error in create bundle upload endpoint"); + let report = ErrorReport::new( + error, + String::from("fake-org-slug"), + Some(String::from("There was an unexpected error")), + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "There was an unexpected error", + "", + "Pass --test-collection-id.", + "", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + ] + ); +} + +#[test] +fn prints_connection_refused_beneath_the_base_message() { + let report = ErrorReport::new( + connection_refused_error(), + String::from("fake-org-slug"), + Some(String::from("There was an unexpected error")), + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "There was an unexpected error", + "", + "Unable to connect to trunk's server", + ] + ); +} + +#[test] +fn prints_connection_refused_without_a_base_message() { + let report = ErrorReport::new( + connection_refused_error(), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + ["Error Encountered", "Unable to connect to trunk's server"] + ); +} + +#[test] +fn prints_unauthorized_with_a_settings_hint() { + let report = ErrorReport::new(unauthorized_error(), String::from("fake-org-slug"), None); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "", + "", + "Unathorized access, your Trunk organization URL slug or token may be incorrect", + " Hint: You can find it under the settings page at https://app.trunk.io/fake-org-slug/settings", + ] + ); +} + +#[test] +fn prints_a_generic_error_without_a_base_message() { + let report = ErrorReport::new( + anyhow::anyhow!("--allow-forked-pr-uploads requires --test-collection-id"), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "--allow-forked-pr-uploads requires --test-collection-id", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + ] + ); +} + +#[test] +fn prints_an_interrupting_error_with_its_exit_code() { + let report = ErrorReport::new( + anyhow::anyhow!(InterruptingError::new( + "No test output files found to validate" + )), + String::from("fake-org-slug"), + None, + ); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "No test output files found to validate", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, returning exit code: 70", + ] + ); +} + +#[test] +fn prints_the_default_exit_code() { + let mut report = ErrorReport::new( + anyhow::anyhow!("something went wrong"), + String::from("fake-org-slug"), + None, + ); + report.context.exit_code = Some(exitcode::OK); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "something went wrong", + "For more help, contact us at https://slack.trunk.io/", + "", + "No errors occurred, returning default exit code: 0", + ] + ); +} + +#[test] +fn prints_any_other_exit_code() { + let mut report = ErrorReport::new( + anyhow::anyhow!("something went wrong"), + String::from("fake-org-slug"), + None, + ); + report.context.exit_code = Some(exitcode::USAGE); + + assert_eq!( + error_summary(&report), + [ + "Error Encountered", + "something went wrong", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, returning exit code: 64", + ] + ); +} diff --git a/cli/tests/summary/main.rs b/cli/tests/summary/main.rs new file mode 100644 index 00000000..ad3c87ed --- /dev/null +++ b/cli/tests/summary/main.rs @@ -0,0 +1,12 @@ +mod error_report; +mod upload; +mod validate; + +fn unauthorized_error() -> anyhow::Error { + let response = http::Response::builder().status(401).body("").unwrap(); + anyhow::Error::from( + reqwest::Response::from(response) + .error_for_status() + .unwrap_err(), + ) +} diff --git a/cli/tests/summary/upload.rs b/cli/tests/summary/upload.rs new file mode 100644 index 00000000..8b5c908d --- /dev/null +++ b/cli/tests/summary/upload.rs @@ -0,0 +1,817 @@ +use std::collections::{BTreeMap, HashMap}; + +use api::{client::ClientErrorExplanation, urls::TestCaseGuidScope}; +use bundle::{ + BundleMeta, BundleMetaBaseProps, BundleMetaDebugProps, BundleMetaJunitProps, + QuarantineResolutionMode, Test, TestCollectionProps, +}; +use context::{ + junit::validator::{ + JunitReportValidation, JunitReportValidationIssueSubOptimal, + JunitTestCaseValidationIssueInvalid, JunitTestCaseValidationIssueSubOptimal, + JunitValidationIssue, JunitValidationIssueType, + }, + repo::{BundleRepo, RepoUrlParts}, +}; +use display::end_output::EndOutput; +use pretty_assertions::assert_eq; +use superconsole::Line; +use trunk_analytics_cli::{ + context_quarantine::{QuarantineContext, QuarantineFetchStatus}, + error_report::ErrorReport, + report_limiting::ValidationReport, + upload_command::UploadRunResult, + validate_command::JunitReportValidations, +}; + +use crate::unauthorized_error; + +const BUNDLE_UPLOAD_ID: &str = "4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d"; +const UPLOAD_ERROR_MESSAGE: &str = + "There was an unexpected error that occurred while uploading test results"; + +fn upload_summary(result: &UploadRunResult) -> Vec { + result + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + .collect() +} + +fn repo() -> RepoUrlParts { + RepoUrlParts { + host: String::from("github.com"), + owner: String::from("trunk-io"), + name: String::from("analytics-cli"), + } +} + +fn upload_result(num_tests: usize) -> UploadRunResult { + let mut quarantine_context = QuarantineContext::skip_fetch(Vec::new()); + quarantine_context.repo = repo(); + quarantine_context.org_url_slug = String::from("test-org"); + UploadRunResult { + error_report: None, + quarantine_context, + meta: BundleMeta { + base_props: BundleMetaBaseProps { + version: String::new(), + cli_version: String::new(), + org: String::from("test-org"), + test_collection: None, + repo: BundleRepo { + repo: repo(), + ..Default::default() + }, + bundle_upload_id: String::from(BUNDLE_UPLOAD_ID), + tags: Vec::new(), + file_sets: Vec::new(), + envs: HashMap::new(), + upload_time_epoch: 0, + test_command: None, + os_info: None, + quarantined_tests: Vec::new(), + codeowners: None, + use_uncloned_repo: None, + quarantine_resolution_mode: QuarantineResolutionMode::default(), + }, + junit_props: BundleMetaJunitProps { + num_files: 1, + num_tests, + }, + debug_props: BundleMetaDebugProps { + command_line: String::new(), + trunk_envs: HashMap::new(), + }, + bundle_upload_id_v2: String::new(), + variant: None, + internal_bundled_file: None, + failed_tests: Vec::new(), + }, + validations: JunitReportValidations::new(BTreeMap::new()), + validation_report: ValidationReport::Limited, + show_failure_messages: false, + test_collection_short_id: None, + hide_test_collection_links: false, + api_address: String::from("https://api.trunk.io"), + guid_scope: None, + } +} + +fn in_collection(mut result: UploadRunResult) -> UploadRunResult { + result.test_collection_short_id = Some(String::from("abc123")); + result.meta.base_props.test_collection = Some(TestCollectionProps { + short_id: String::from("abc123"), + bundle_meta_id: String::from("0b6f4c1e-8a7d-4e2b-9c3f-5d1a2b3c4d5e"), + bundle_meta_created_at: String::from("2026-10-21T12:00:00Z"), + }); + result +} + +fn failure(name: &str, parent_name: &str, file: Option<&str>) -> Test { + Test { + name: String::from(name), + parent_name: String::from(parent_name), + class_name: None, + file: file.map(String::from), + id: format!("{name}-id"), + timestamp_millis: None, + is_quarantined: false, + failure_message: None, + variant: None, + } +} + +fn warning(issue: JunitTestCaseValidationIssueSubOptimal) -> JunitValidationIssueType { + JunitValidationIssueType::TestCase(JunitValidationIssue::SubOptimal(issue)) +} + +fn report_warning(issue: JunitReportValidationIssueSubOptimal) -> JunitValidationIssueType { + JunitValidationIssueType::Report(JunitValidationIssue::SubOptimal(issue)) +} + +fn name_too_short() -> JunitValidationIssueType { + JunitValidationIssueType::TestCase(JunitValidationIssue::Invalid( + JunitTestCaseValidationIssueInvalid::TestCaseNameTooShort(String::new()), + )) +} + +fn validation(all_issues: Vec) -> anyhow::Result { + Ok(JunitReportValidation { + all_issues, + ..Default::default() + }) +} + +fn validations() -> JunitReportValidations { + JunitReportValidations::new(BTreeMap::from([ + ( + String::from("test-results/a.xml"), + validation(vec![ + name_too_short(), + name_too_short(), + report_warning( + JunitReportValidationIssueSubOptimal::TestCasesFileOrFilepathMissing, + ), + report_warning(JunitReportValidationIssueSubOptimal::MissingTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::FutureTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::OldTimestamps), + report_warning(JunitReportValidationIssueSubOptimal::StaleTimestamps), + warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration), + warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseClassnameTooShort(String::new()), + ), + warning(JunitTestCaseValidationIssueSubOptimal::TestCaseNameTooLong( + String::new(), + )), + warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseFileOrFilepathTooLong( + String::new(), + ), + ), + ]), + ), + ( + String::from("test-results/b.xml"), + Err(anyhow::anyhow!("unexpected end of file")), + ), + ( + String::from("test-results/c.xml"), + validation(vec![name_too_short()]), + ), + ( + String::from("test-results/d.xml"), + validation(vec![warning( + JunitTestCaseValidationIssueSubOptimal::TestCaseNoTimeDuration, + )]), + ), + (String::from("test-results/e.xml"), validation(Vec::new())), + (String::from("test-results/f.xml"), validation(Vec::new())), + (String::from("test-results/g.xml"), validation(Vec::new())), + (String::from("test-results/h.xml"), validation(Vec::new())), + ( + String::from("test-results/i.xml"), + validation(vec![report_warning( + JunitReportValidationIssueSubOptimal::MissingTimestamps, + )]), + ), + ])) +} + +#[test] +fn prints_a_passing_run() { + let result = upload_result(5); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_that_no_tests_were_found() { + let result = upload_result(0); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 0 Pass: 0 Fail: 0 Quarantined: 0 Pass Ratio: N/A", + "", + "⚠️ No tests were found in the provided test results", + ] + ); +} + +#[test] +fn prints_failures_that_were_all_quarantined() { + let mut result = upload_result(4); + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .group_is_quarantined = true; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![ + failure("adds", "MathTests", Some("src/math.rs")), + failure("subtracts", "MathTests", Some("src/math.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 4 Pass: 2 Fail: 0 Quarantined: 2 Pass Ratio: 50.0%", + "", + "β€οΈβ€πŸ©Ή 2 tests failed and were quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "πŸŽ‰ All test failures were quarantined, overriding exit code to be exit_success (0)", + ] + ); +} + +#[test] +fn prints_failures_that_were_not_quarantined() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + result.quarantine_context.fetch_status = QuarantineFetchStatus::FetchSucceeded; + result + .quarantine_context + .quarantine_status + .quarantine_results = vec![failure("adds", "MathTests", Some("src/math.rs"))]; + result.quarantine_context.failures = vec![ + failure("subtracts", "MathTests", Some("src/math.rs")), + failure("divides", "MathTests", Some("src/math.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 2 Fail: 2 Quarantined: 1 Pass Ratio: 40.0%", + "", + "β€οΈβ€πŸ©Ή 1 test failed and was quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "❌ 2 tests failed and were not quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/divides", + " β€· https://app.trunk.io/test-org/flaky-tests/test/divides-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn prints_a_warning_when_the_quarantine_lookup_failed() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.quarantine_context.fetch_status = + QuarantineFetchStatus::FetchFailed(anyhow::anyhow!("request timed out")); + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", + " We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn stops_at_an_error_that_blocks_quarantining() { + let mut result = upload_result(3); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + unauthorized_error(), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_summary(&result), + [ + "", + "Error Encountered", + "There was an unexpected error that occurred while uploading test results", + "", + "Unathorized access, your Trunk organization URL slug or token may be incorrect", + " Hint: You can find it under the settings page at https://app.trunk.io/test-org/settings", + ] + ); +} + +#[test] +fn proceeds_past_an_upload_error() { + let mut result = upload_result(3); + result.meta.base_props.bundle_upload_id = String::new(); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + result.error_report = Some(ErrorReport::new( + anyhow::anyhow!("HTTP status client error (400 Bad Request)") + .context(ClientErrorExplanation { + code: Some(String::from("SOME_CODE")), + message: Some(String::from("Pass --test-collection-id.")), + }) + .context("Error in create bundle upload endpoint"), + String::from("test-org"), + Some(String::from(UPLOAD_ERROR_MESSAGE)), + )); + + assert_eq!( + upload_summary(&result), + [ + "", + "Error Encountered", + "There was an unexpected error that occurred while uploading test results", + "", + "Pass --test-collection-id.", + "", + "For more help, contact us at https://slack.trunk.io/", + "", + "Errors occurred during execution, using quarantining exit code", + "Proceeding with test report...", + "", + "πŸ“š Test Report", + " Total: 3 Pass: 2 Fail: 1 Quarantined: 0 Pass Ratio: 66.7%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn prints_a_limited_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "❕ 9 files found, 5 with issues", + "", + " ❌ test-results/a.xml Has Errors And Warnings", + " β†ͺ ❌ Errors", + " β†ͺ test case name too short", + " β†ͺ test case name too short", + " β†ͺ ⚠️ Warnings", + " β†ͺ report has test cases with missing file or filepath", + " β†ͺ report has test cases with missing timestamp", + " β†ͺ report has test cases with future timestamp", + " β†ͺ report has old (> 24 hour(s)) timestamps", + " β†ͺ report has stale (> 1 hour(s)) timestamps", + " β†ͺ test case or parent has no time duration", + " β†ͺ test case classname too short", + " β†ͺ test case name too long, truncated to 1000", + " ❌ test-results/c.xml Has Errors", + " β†ͺ test case name too short", + " ⚠️ test-results/d.xml Has Warnings", + " β†ͺ test case or parent has no time duration", + "…and 1 more", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_a_full_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + result.validation_report = ValidationReport::Full; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "❕ 9 files found, 5 with issues", + "", + " ❌ test-results/a.xml Has Errors And Warnings", + " β†ͺ ❌ Errors", + " β†ͺ test case name too short", + " β†ͺ test case name too short", + " β†ͺ ⚠️ Warnings", + " β†ͺ report has test cases with missing file or filepath", + " β†ͺ report has test cases with missing timestamp", + " β†ͺ report has test cases with future timestamp", + " β†ͺ report has old (> 24 hour(s)) timestamps", + " β†ͺ report has stale (> 1 hour(s)) timestamps", + " β†ͺ test case or parent has no time duration", + " β†ͺ test case classname too short", + " β†ͺ test case name too long, truncated to 1000", + " β†ͺ test case file or filepath too long", + " ❌ test-results/c.xml Has Errors", + " β†ͺ test case name too short", + " ⚠️ test-results/d.xml Has Warnings", + " β†ͺ test case or parent has no time duration", + " ⚠️ test-results/i.xml Has Warnings", + " β†ͺ report has test cases with missing timestamp", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_a_blank_line_for_no_validation_report() { + let mut result = upload_result(5); + result.validations = validations(); + result.validation_report = ValidationReport::None; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_validation_when_every_file_is_correct() { + let mut result = upload_result(5); + result.validations = JunitReportValidations::new(BTreeMap::from([( + String::from("test-results/a.xml"), + validation(Vec::new()), + )])); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“‚ File Validation", + "", + "βœ… 1 file found, all fully correct", + "", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn prints_long_failure_messages_when_asked() { + let mut result = upload_result(1); + result.show_failure_messages = true; + result.quarantine_context.exit_code = 1; + let mut message_lines = vec![ + String::from("assertion `left == right` failed\r"), + String::from("\tleft: 1"), + String::new(), + String::from(" "), + String::from("\tright: 2"), + ]; + message_lines.extend((6..=23).map(|n| format!("frame {n}"))); + result.quarantine_context.failures = vec![Test { + failure_message: Some(message_lines.join("\n")), + ..failure("adds", "MathTests", None) + }]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 1 Pass: 0 Fail: 1 Quarantined: 0 Pass Ratio: 0.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " Failure: ", + " assertion `left == right` failed", + " left: 1", + " right: 2", + " frame 6", + " frame 7", + " frame 8", + " frame 9", + " frame 10", + " frame 11", + " frame 12", + " frame 13", + " frame 14", + " frame 15", + " frame 16", + " frame 17", + " frame 18", + " frame 19", + " frame 20", + " …and 3 more lines not shown", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn prints_the_upload_as_a_collection_link() { + let mut result = in_collection(upload_result(2)); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![Test { + id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), + ..failure("adds", "MathTests", None) + }]; + result.guid_scope = Some(TestCaseGuidScope { + test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), + repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), + }); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/tests/81d0929e-fc06-8b61-8506-991d8ebc51ea", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn links_tests_by_short_link_without_collection_ids() { + let mut result = in_collection(upload_result(2)); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![failure("adds", "MathTests", None)]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Upload: https://app.trunk.io/test-org/flaky-tests/collections/abc123/uploads/eyJpZCI6IjBiNmY0YzFlLThhN2QtNGUyYi05YzNmLTVkMWEyYjNjNGQ1ZSIsImNyZWF0ZWRBdCI6MTc5MjU4NDAwMDAwMH0", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/collections/abc123/t/adds-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn prints_the_bundle_upload_id_when_collection_links_are_hidden() { + let mut result = in_collection(upload_result(2)); + result.hide_test_collection_links = true; + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![Test { + id: String::from("c33a7f64-8f3e-5db9-b37b-2ea870d2441b"), + ..failure("adds", "MathTests", None) + }]; + result.guid_scope = Some(TestCaseGuidScope { + test_collection_id: String::from("9a8b7c6d-5e4f-4a3b-8c2d-1e0f9a8b7c6d"), + repo_id: String::from("1f2e3d4c-5b6a-4978-8a6b-5c4d3e2f1a0b"), + }); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 2 Pass: 1 Fail: 1 Quarantined: 0 Pass Ratio: 50.0%", + "", + "❌ 1 test failed and was not quarantined", + "", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/c33a7f64-8f3e-5db9-b37b-2ea870d2441b?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn omits_the_upload_line_without_a_bundle_upload_id() { + let mut result = upload_result(5); + result.meta.base_props.bundle_upload_id = String::new(); + + assert_eq!( + upload_summary(&result), + [ + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "πŸŽ‰ No test failures found!", + ] + ); +} + +#[test] +fn groups_failures_by_file_then_suite_then_neither() { + let mut result = upload_result(10); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = vec![ + failure("parses_dates", "DateTests", Some("src/date.rs")), + failure("adds", "MathTests", None), + failure("loads", "", None), + failure("parses_times", "TimeTests", Some("src/date.rs")), + failure("subtracts", "MathTests", None), + failure("boots", "", None), + failure("formats", "", Some("src/a.rs")), + ]; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 10 Pass: 3 Fail: 7 Quarantined: 0 Pass Ratio: 30.0%", + "", + "❌ 7 tests failed and were not quarantined", + "", + " πŸ“ src/a.rs", + " formats", + " β€· https://app.trunk.io/test-org/flaky-tests/test/formats-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ“ src/date.rs", + " DateTests/parses_dates", + " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_dates-id?repo=trunk-io%2Fanalytics-cli", + " TimeTests/parses_times", + " β€· https://app.trunk.io/test-org/flaky-tests/test/parses_times-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ“¦ MathTests", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " ", + " πŸ” Other", + " loads", + " β€· https://app.trunk.io/test-org/flaky-tests/test/loads-id?repo=trunk-io%2Fanalytics-cli", + " boots", + " β€· https://app.trunk.io/test-org/flaky-tests/test/boots-id?repo=trunk-io%2Fanalytics-cli", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn shows_three_failures_per_group() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + result.quarantine_context.failures = ["adds", "subtracts", "multiplies", "divides", "rounds"] + .into_iter() + .map(|name| failure(name, "MathTests", Some("src/math.rs"))) + .collect(); + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 0 Fail: 5 Quarantined: 0 Pass Ratio: 0.0%", + "", + "❌ 5 tests failed and were not quarantined", + "", + " πŸ“ src/math.rs", + " MathTests/adds", + " β€· https://app.trunk.io/test-org/flaky-tests/test/adds-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/subtracts", + " β€· https://app.trunk.io/test-org/flaky-tests/test/subtracts-id?repo=trunk-io%2Fanalytics-cli", + " MathTests/multiplies", + " β€· https://app.trunk.io/test-org/flaky-tests/test/multiplies-id?repo=trunk-io%2Fanalytics-cli", + " …and 2 more failures in this group", + " ", + "", + "⚠️ Some test failures were not quarantined, using exit code: 1", + ] + ); +} + +#[test] +fn warns_about_a_non_zero_exit_code_without_failures() { + let mut result = upload_result(5); + result.quarantine_context.exit_code = 1; + + assert_eq!( + upload_summary(&result), + [ + "🏷️ Bundle Upload ID: 4b1d2c3e-5f60-4a7b-8c9d-0e1f2a3b4c5d", + "", + "πŸ“š Test Report", + " Total: 5 Pass: 5 Fail: 0 Quarantined: 0 Pass Ratio: 100.0%", + "", + "⚠️ No test failures found, but non zero exit code provided: 1", + "This may indicate that some tests were not run or that there were other issues during the test run.", + ] + ); +} diff --git a/cli/tests/summary/validate.rs b/cli/tests/summary/validate.rs new file mode 100644 index 00000000..f5ff3cbb --- /dev/null +++ b/cli/tests/summary/validate.rs @@ -0,0 +1,110 @@ +use clap::Parser; +use display::end_output::EndOutput; +use pretty_assertions::assert_eq; +use superconsole::Line; +use trunk_analytics_cli::validate_command::{ValidateArgs, run_validate}; + +#[derive(Parser)] +struct Validate { + #[command(flatten)] + args: ValidateArgs, +} + +async fn validate_summary(args: &[&str]) -> Vec { + let Validate { args } = Validate::parse_from(["validate"].iter().chain(args)); + run_validate(args) + .await + .unwrap() + .output() + .unwrap() + .iter() + .map(Line::to_unstyled) + // CODEOWNERS is printed as an absolute path. + .map(|line| line.replace(env!("CARGO_MANIFEST_DIR"), "")) + .collect() +} + +#[tokio::test] +async fn prints_validation_errors_warnings_and_codeowners() { + assert_eq!( + validate_summary(&[ + "--junit-paths", + "test_fixtures/summary/validate/*.xml", + "--codeowners-path", + "test_fixtures/summary/validate", + ]) + .await, + [ + "πŸ“š Validating 4 files:", + " Files matching: test_fixtures/summary/validate/*.xml:", + " test_fixtures/summary/validate/broken.xml", + " test_fixtures/summary/validate/empty.xml", + " test_fixtures/summary/validate/multiple.xml", + " test_fixtures/summary/validate/unit.xml", + "", + "File parse issues", + "test_fixtures/summary/validate/broken.xml", + "1 errors, 0 warnings", + " ❌ Errors:", + " ill-formed document: expected ``, but `` was foundCaused by: expected ``, but `` was found", + "", + "test_fixtures/summary/validate/empty.xml", + "0 errors, 1 warnings", + " ⚠️ Warnings:", + " no reports found", + "", + "test_fixtures/summary/validate/multiple.xml", + "1 errors, 0 warnings", + " ❌ Errors:", + " multiple reports found", + "", + "Test Validation Issues", + "test_fixtures/summary/validate/unit.xml", + "1 test suites, 3 test cases, 1 errors, 3 warnings", + " ❌ Errors:", + " test case name too short", + " ⚠️ Warnings:", + " report has test cases with missing file or filepath", + " report has test cases with missing timestamp", + " test case or parent has no time duration", + "", + "Checking for codeowners file...", + " Found codeowners path: \"/test_fixtures/summary/validate/CODEOWNERS\"", + " CODEOWNERS found but test cases are missing filepaths. We will not be able to correlate flaky tests with owners.", + "", + "0 valid files, 2 files with warnings, and 3 files with errors, with 4 files total (a file is double counted if it has both errors and warnings)", + "4 warnings, and 3 errors", + ] + ); +} + +#[tokio::test] +async fn prints_bep_parse_errors_without_a_codeowners_file() { + assert_eq!( + validate_summary(&[ + "--bazel-bep-path", + "test_fixtures/summary/bep/bep_with_parse_error" + ]) + .await, + [ + "⚠️ BEP file had parse errors: [\"Error parsing build event: EOF while parsing a value at line 3 column 0\"]", + "", + "πŸ“š Validating 1 files:", + " Files matching: test_fixtures/summary/bep/test.xml:", + " test_fixtures/summary/bep/test.xml", + "", + "File parse issues", + "Test Validation Issues", + "test_fixtures/summary/bep/test.xml", + "1 test suites, 1 test cases, 0 errors, 1 warnings", + " ⚠️ Warnings:", + " report has test cases with missing timestamp", + "", + "Checking for codeowners file...", + " No codeowners file found", + "", + "0 valid files, 1 file with warnings, and 0 files with errors, with 1 file total (a file is double counted if it has both errors and warnings)", + "1 warning, and 0 errors", + ] + ); +}