diff --git a/crates/sqlload-testkit/src/fake_sheets/_helpers/routing.rs b/crates/sqlload-testkit/src/fake_sheets/_helpers/routing.rs index a8bca63d..8c12e831 100644 --- a/crates/sqlload-testkit/src/fake_sheets/_helpers/routing.rs +++ b/crates/sqlload-testkit/src/fake_sheets/_helpers/routing.rs @@ -18,15 +18,30 @@ fn google_error(code: u16, status: &str, message: &str) -> Value { json!({"error": {"code": code, "message": message, "status": status}}) } -/// Route one request; every read needs a bearer token the fake handed out and has not refused. +/// Route one request: a token request gets a token unless its refresh token is revoked; every read needs a bearer token the fake handed out and has not refused. pub(crate) async fn handle( State(state): State, parts: Parts, - _body: Bytes, + body: Bytes, ) -> Response { let path = parts.uri.path().to_owned(); if parts.method == axum::http::Method::POST && path == TOKEN_PATH { + let form = query_pairs(&String::from_utf8_lossy(&body)); + let field = |name: &str| { + form.iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.clone()) + .unwrap_or_default() + }; let mut locked = state.lock(); + locked.grants.push(field("grant_type")); + if locked + .revoked_refresh_tokens + .contains(&field("refresh_token")) + { + let body = json!({"error": "invalid_grant", "error_description": "Token has been expired or revoked."}); + return (StatusCode::BAD_REQUEST, body.to_string()).into_response(); + } locked.tokens += 1; let token = format!("{}{}", &TOKEN_PREFIX["Bearer ".len()..], locked.tokens); return ( diff --git a/crates/sqlload-testkit/src/fake_sheets/main/start_fake_sheets.rs b/crates/sqlload-testkit/src/fake_sheets/main/start_fake_sheets.rs index 1c52bae2..63343e2b 100644 --- a/crates/sqlload-testkit/src/fake_sheets/main/start_fake_sheets.rs +++ b/crates/sqlload-testkit/src/fake_sheets/main/start_fake_sheets.rs @@ -1,7 +1,7 @@ //! Start the fake Sheets API. use crate::fake_sheets::_helpers::routing::{fake_key, handle}; -use crate::fake_sheets::constants::SPREADSHEETS_PATH; +use crate::fake_sheets::constants::{SPREADSHEETS_PATH, TOKEN_PATH}; use crate::fake_sheets::models::{FakeSheets, SharedSheets}; /// Serve the fake on a free local port, with no spreadsheets. @@ -23,6 +23,7 @@ pub async fn start_fake_sheets() -> Result { Ok(FakeSheets { key_json: fake_key(&root)?, base_url: format!("{root}{SPREADSHEETS_PATH}"), + token_url: format!("{root}{TOKEN_PATH}"), state, }) } diff --git a/crates/sqlload-testkit/src/fake_sheets/models.rs b/crates/sqlload-testkit/src/fake_sheets/models.rs index 07261d15..c2257d48 100644 --- a/crates/sqlload-testkit/src/fake_sheets/models.rs +++ b/crates/sqlload-testkit/src/fake_sheets/models.rs @@ -23,6 +23,10 @@ pub struct FakeSheetsState { pub refused_tokens: Vec, /// Each read: the range asked for, or `*` for every tab. pub reads: Vec, + /// Refresh tokens the token endpoint refuses as revoked, as Google does with `invalid_grant`. + pub revoked_refresh_tokens: Vec, + /// The grant type of each token request. + pub grants: Vec, } /// Shared fake state. @@ -39,10 +43,12 @@ impl SharedSheets { } } -/// A running fake: the spreadsheets root a connection's `base_url` takes, a service-account key exchanging tokens with it, and its state. +/// A running fake: the spreadsheets root a connection's `base_url` takes, its token endpoint, a service-account key exchanging tokens with it, and its state. #[derive(Debug, Clone)] pub struct FakeSheets { pub base_url: String, + /// The token endpoint, as a Google user connection's `token_url` names it. + pub token_url: String, pub key_json: String, pub state: SharedSheets, } diff --git a/crates/sqlload/src/auth/_helpers/google_oauth.rs b/crates/sqlload/src/auth/_helpers/google_oauth.rs new file mode 100644 index 00000000..c84d34cc --- /dev/null +++ b/crates/sqlload/src/auth/_helpers/google_oauth.rs @@ -0,0 +1,252 @@ +//! A Google user's access tokens for an installed app's OAuth client: one consent in the browser, the refresh token it gives kept, and access tokens renewed with it. + +use crate::auth::_helpers::oauth::{Pkce, now_secs, renew_after}; +use crate::auth::_helpers::sign_in::{Consent, SignInIo, browser_code, say}; +use crate::auth::_helpers::token_cache::{read_entry, write_entry}; +use crate::auth::constants::{ + DEFAULT_GOOGLE_TOKEN_SECS, GOOGLE_ERROR_CHARS, GOOGLE_PROVIDER, INVALID_GRANT, +}; +use crate::auth::errors::TokenError; +use crate::auth::models::{GoogleUser, OauthTokens, Secret}; +use crate::http::constants::TOO_MANY_REQUESTS; +use crate::http::errors::ApiError; +use crate::time_text::main::now_ms::now_ms; + +/// The access token to send now, other than `refused`: the one `held`, while it has a minute left; else, with the connection's `refresh_token`, a renewed one; else the cached one, or one renewed with the cached refresh token, or one from a new consent in the browser through `io`, which is cached. A refresh token Google refuses is an error naming why, as is a consent without a terminal. +pub(crate) async fn google_access_token( + http: &reqwest::Client, + user: &GoogleUser, + held: &tokio::sync::Mutex>, + (refused, io): (Option<&Secret>, &SignInIo), +) -> Result { + let mut held = held.lock().await; + let now = now_ms(); + let current = held + .as_ref() + .filter(|(token, expires)| *expires > now && Some(token) != refused); + if let Some((token, _)) = current { + return Ok(token.clone()); + } + let tokens = match &user.refresh_token { + Some(configured) => renew(http, user, configured) + .await + .map_err(|error| configured_refused(user, error))?, + None => cached_or_consent(http, user, (refused, io)).await?, + }; + let expires = i64::try_from(tokens.renew_after) + .unwrap_or(i64::MAX / 1000) + .saturating_mul(1000); + *held = Some((tokens.access_token.clone(), expires)); + Ok(tokens.access_token) +} + +/// The cached access token when it is not `refused` and has a minute left; else one renewed with the cached refresh token; else a new consent's. New tokens are cached. +async fn cached_or_consent( + http: &reqwest::Client, + user: &GoogleUser, + (refused, io): (Option<&Secret>, &SignInIo), +) -> Result { + let key = cache_key(user); + let cached = read_entry(&user.cache_file, &key); + let now = now_secs(); + if let Some(cached) = cached + .as_ref() + .filter(|cached| cached.renew_after > now && Some(&cached.access_token) != refused) + { + return Ok(cached.clone()); + } + let refresh = cached.and_then(|cached| { + cached.refresh_token.filter(|_| { + cached + .refresh_expires_at + .is_none_or(|expires| expires > now) + }) + }); + let renewed = match refresh { + Some(refresh) => match renew(http, user, &refresh).await { + Ok(renewed) => Ok(renewed), + Err(TokenError::Refused(reason)) => Err(Some(reason)), + Err(TokenError::Failed(error)) => return Err(error), + }, + None => Err(None), + }; + let tokens = match renewed { + Ok(renewed) => renewed, + Err(refusal) => consent(http, user, io).await.map_err(|error| match refusal { + Some(reason) => ApiError::Auth(format!( + "Google refused the cached refresh token of connection {:?} ({reason}), so it needs a new sign-in: {error}", + user.connection + )), + None => error, + })?, + }; + if let Err(error) = write_entry(&user.cache_file, &key, &tokens) { + tracing::warn!(%error, "cannot cache the Google sign-in; the next run signs in again"); + } + Ok(tokens) +} + +/// The key a connection's Google sign-in is cached under: its OAuth client and its name. +pub(crate) fn cache_key(user: &GoogleUser) -> String { + format!("{} {}", user.client_id, user.connection) +} + +/// The connection's own refresh token's failure, saying how to replace a refused one. +fn configured_refused(user: &GoogleUser, error: TokenError) -> ApiError { + match error { + TokenError::Refused(reason) => ApiError::Auth(format!( + "Google refused the refresh_token of connection {:?} ({reason}): it was revoked, has expired (an OAuth client whose consent screen is in testing gets tokens that last 7 days), or belongs to another client; get a new one by signing in from a terminal with refresh_token unset", + user.connection + )), + TokenError::Failed(error) => error, + } +} + +/// Tokens renewed with `refresh`, which they keep, as Google sends no new one. +async fn renew( + http: &reqwest::Client, + user: &GoogleUser, + refresh: &Secret, +) -> Result { + let grant = [ + ("grant_type", "refresh_token"), + ("refresh_token", refresh.expose()), + ]; + let mut renewed = token_grant(http, user, &grant).await?; + if renewed.refresh_token.is_none() { + renewed.refresh_token = Some(refresh.clone()); + } + Ok(renewed) +} + +/// Have the person consent in the browser and exchange the code for tokens, which must hold a refresh token. +async fn consent( + http: &reqwest::Client, + user: &GoogleUser, + io: &SignInIo, +) -> Result { + let who = format!("the Google account for connection {:?}", user.connection); + let consent = Consent { + provider: GOOGLE_PROVIDER, + who: &who, + redirect_uri: None, + unattended: "run `sqlload plan` from a terminal once to sign in, or set the connection's refresh_token for unattended runs", + }; + let signed = browser_code(&consent, io, |redirect_uri, pkce| { + authorize_url(user, redirect_uri, pkce) + }) + .await?; + let grant = [ + ("grant_type", "authorization_code"), + ("code", signed.code.as_str()), + ("redirect_uri", signed.redirect_uri.as_str()), + ("code_verifier", signed.verifier.as_str()), + ]; + let tokens = token_grant(http, user, &grant) + .await + .map_err(|error| match error { + TokenError::Refused(reason) => { + ApiError::Auth(format!("Google refused the sign-in code: {reason}")) + } + TokenError::Failed(error) => ApiError::Auth(format!( + "exchanging the sign-in code failed ({error}); sign in again" + )), + })?; + if tokens.refresh_token.is_none() { + return Err(ApiError::Auth( + "Google sent no refresh token for the sign-in; remove SQLLoad's access at https://myaccount.google.com/permissions and sign in again".to_owned(), + )); + } + say(&format!( + "Signed in to Google for connection {:?}; the refresh token is cached in {}.", + user.connection, + user.cache_file.display() + )); + Ok(tokens) +} + +/// The consent page's address: the user's scope, offline so Google sends a refresh token, with the consent screen shown each time so it always does, and the account chooser so another account can sign in. +fn authorize_url(user: &GoogleUser, redirect_uri: &str, pkce: &Pkce) -> Result { + let mut url = reqwest::Url::parse(&user.authorize_url) + .map_err(|error| ApiError::Auth(format!("bad Google authorization URL: {error}")))?; + url.query_pairs_mut() + .append_pair("client_id", &user.client_id) + .append_pair("response_type", "code") + .append_pair("redirect_uri", redirect_uri) + .append_pair("scope", &user.scope) + .append_pair("code_challenge", &pkce.challenge) + .append_pair("code_challenge_method", "S256") + .append_pair("state", &pkce.state) + .append_pair("access_type", "offline") + .append_pair("prompt", "consent select_account"); + Ok(url.into()) +} + +/// Send one grant, with the client's ID and secret, to Google's token endpoint. `invalid_grant` is a refusal; throttling and server errors keep their status, so they are retried; any other refusal names the client. +async fn token_grant( + http: &reqwest::Client, + user: &GoogleUser, + grant: &[(&str, &str)], +) -> Result { + let mut form: Vec<(&str, &str)> = grant.to_vec(); + form.push(("client_id", &user.client_id)); + form.push(("client_secret", user.client_secret.expose())); + let response = http + .post(&user.token_url) + .form(&form) + .send() + .await + .map_err(|error| TokenError::Failed(error.into()))?; + let status = response.status(); + let text = response + .text() + .await + .map_err(|error| TokenError::Failed(error.into()))?; + let body: serde_json::Value = match serde_json::from_str(&text) { + Ok(body) => body, + Err(_) => serde_json::Value::Null, + }; + let error = body["error"].as_str().unwrap_or(status.as_str()).to_owned(); + let description: String = body["error_description"] + .as_str() + .unwrap_or(&text) + .chars() + .take(GOOGLE_ERROR_CHARS) + .collect(); + match status.as_u16() { + _ if status.is_success() => tokens_from(&body).map_err(TokenError::Failed), + _ if error == INVALID_GRANT => Err(TokenError::Refused(format!("{error}: {description}"))), + code if code == TOO_MANY_REQUESTS || status.is_server_error() => { + Err(TokenError::Failed(ApiError::Status { + status: code, + code: String::new(), + message: format!("Google's token endpoint: {error}: {description}"), + })) + } + _ => Err(TokenError::Failed(ApiError::Auth(format!( + "Google's token endpoint refused OAuth client {} ({error}: {description}); check the connection's client_id and client_secret", + user.client_id + )))), + } +} + +/// The tokens in a token endpoint answer. +fn tokens_from(body: &serde_json::Value) -> Result { + let access_token = body["access_token"].as_str().ok_or_else(|| { + ApiError::Protocol("Google's token endpoint sent no access_token".to_owned()) + })?; + let now = now_secs(); + let lifetime = body["expires_in"] + .as_u64() + .unwrap_or(DEFAULT_GOOGLE_TOKEN_SECS); + Ok(OauthTokens { + access_token: Secret(access_token.to_owned()), + renew_after: renew_after(now, Some(lifetime)), + refresh_token: body["refresh_token"] + .as_str() + .map(|token| Secret(token.to_owned())), + refresh_expires_at: body["refresh_token_expires_in"] + .as_u64() + .map(|seconds| now.saturating_add(seconds)), + }) +} diff --git a/crates/sqlload/src/auth/_helpers/mod.rs b/crates/sqlload/src/auth/_helpers/mod.rs index f2372e02..f86e0315 100644 --- a/crates/sqlload/src/auth/_helpers/mod.rs +++ b/crates/sqlload/src/auth/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod google_oauth; pub(crate) mod jwt; pub(crate) mod oauth; pub(crate) mod sign_in; diff --git a/crates/sqlload/src/auth/_helpers/oauth.rs b/crates/sqlload/src/auth/_helpers/oauth.rs index debca0ae..79d99a3f 100644 --- a/crates/sqlload/src/auth/_helpers/oauth.rs +++ b/crates/sqlload/src/auth/_helpers/oauth.rs @@ -112,8 +112,11 @@ pub(crate) fn redirect_state(redirected: &str) -> Option { } } -/// The authorization code in the address the sign-in redirected to, checked against `state`; Snowflake's own error when it sent one, with control characters dropped so it cannot drive the terminal. -pub(crate) fn redirect_code(redirected: &str, state: &str) -> Result { +/// The authorization code in the address the sign-in redirected to, checked against `state`; `provider`'s own error when it sent one, with control characters dropped so it cannot drive the terminal. +pub(crate) fn redirect_code( + redirected: &str, + (state, provider): (&str, &str), +) -> Result { let url = redirect_url(redirected)?; let printable = |text: String| -> String { text.chars() @@ -123,7 +126,7 @@ pub(crate) fn redirect_code(redirected: &str, state: &str) -> Result SignInIo { } } +/// A browser sign-in to one provider: who signs in, the redirect URI an OAuth client registered (a free loopback port when `None`), and what to use instead when no terminal is there. +pub(crate) struct Consent<'a> { + pub(crate) provider: &'a str, + pub(crate) who: &'a str, + pub(crate) redirect_uri: Option<&'a str>, + pub(crate) unattended: &'a str, +} + +/// What a browser sign-in returned: the authorization code, the redirect URI it was sent to, and the PKCE verifier that goes with it. +pub(crate) struct SignedCode { + pub(crate) code: String, + pub(crate) redirect_uri: String, + pub(crate) verifier: String, +} + /// Sign in through the browser and exchange the code for tokens. Without a terminal this fails at once, as nobody is there to sign in. pub(crate) async fn sign_in(client: &OauthClient, io: &SignInIo) -> Result { + let unattended = "run `sqlload debug` from a terminal once, or use auth = { method = \"key_pair\" } or { method = \"pat\" } for unattended runs"; + let consent = Consent { + provider: SNOWFLAKE_PROVIDER, + who: &client.user, + redirect_uri: client.redirect_uri.as_deref(), + unattended, + }; + let signed = browser_code(&consent, io, |redirect_uri, pkce| { + authorize_url(client, redirect_uri, pkce) + }) + .await?; + let grant = [ + ("grant_type", "authorization_code"), + ("code", signed.code.as_str()), + ("redirect_uri", signed.redirect_uri.as_str()), + ("code_verifier", signed.verifier.as_str()), + ]; + let tokens = token_request(client, &grant) + .await + .map_err(|error| match error { + TokenError::Refused(reason) => { + ApiError::Auth(format!("Snowflake refused the sign-in code: {reason}")) + } + TokenError::Failed(error) => error, + })?; + say(&format!("Signed in to Snowflake as {}.", client.user)); + Ok(tokens) +} + +/// Have the person sign in to `consent.provider` in the browser, at the address `authorize` builds for a redirect URI and PKCE pair, and return the code it redirected back with: by a redirect to a loopback port, or by the person pasting the address the browser ended on. Without a terminal this fails at once. +pub(crate) async fn browser_code( + consent: &Consent<'_>, + io: &SignInIo, + authorize: impl FnOnce(&str, &Pkce) -> Result, +) -> Result { + let Consent { provider, who, .. } = consent; if !(io.terminal)() { return Err(ApiError::Auth(format!( - "no usable Snowflake sign-in is cached for {}, and signing in in the browser needs a terminal; run `sqlload debug` from a terminal once, or use auth = {{ method = \"key_pair\" }} or {{ method = \"pat\" }} for unattended runs", - client.user + "no usable {provider} sign-in is cached for {who}, and signing in in the browser needs a terminal; {}", + consent.unattended ))); } let pkce = new_pkce()?; - let (listener, redirect_uri) = loopback(client.redirect_uri.as_deref()).await?; - let url = authorize_url(client, &redirect_uri, &pkce)?; + let (listener, redirect_uri) = loopback(consent.redirect_uri).await?; + let url = authorize(&redirect_uri, &pkce)?; say(&format!( - "Sign in to Snowflake as {} in your browser:\n {url}", - client.user + "Sign in to {provider} as {who} in your browser:\n {url}" )); let redirected = match (io.open_browser)(&url) { true => { say("Waiting for the browser to finish signing in..."); tokio::time::timeout( Duration::from_secs(SIGN_IN_TIMEOUT_SECS), - receive_redirect(&listener, &pkce.state), + receive_redirect(&listener, (&pkce.state, provider)), ) .await .map_err(|_| { @@ -85,23 +135,11 @@ pub(crate) async fn sign_in(client: &OauthClient, io: &SignInIo) -> Result { - ApiError::Auth(format!("Snowflake refused the sign-in code: {reason}")) - } - TokenError::Failed(error) => error, - })?; - say(&format!("Signed in to Snowflake as {}.", client.user)); - Ok(tokens) + Ok(SignedCode { + code: redirect_code(&redirected, (&pkce.state, provider))?, + redirect_uri, + verifier: pkce.verifier, + }) } /// A listener on the loopback address and the redirect URI that reaches it: the configured one (which must be `http://127.0.0.1[:port][/path]`), else a free port. @@ -142,13 +180,20 @@ async fn loopback(configured: Option<&str>) -> Result<(TcpListener, String), Api } /// The path and query of the first request to `listener` that carries this sign-in's `state`. Each connection is read on its own task with a time limit, so one that sends nothing or garbage holds nothing up; a request for another sign-in or with no state (another local process, or a web page probing the port) gets a 400 and is otherwise ignored. -async fn receive_redirect(listener: &TcpListener, state: &str) -> Result { +async fn receive_redirect( + listener: &TcpListener, + (state, provider): (&str, &str), +) -> Result { let (found, mut received) = mpsc::channel::(1); loop { tokio::select! { accepted = listener.accept() => match accepted { Ok((stream, _)) => { - tokio::spawn(answer_redirect(stream, state.to_owned(), found.clone())); + tokio::spawn(answer_redirect( + stream, + (state.to_owned(), provider.to_owned()), + found.clone(), + )); } Err(error) => { tracing::debug!(%error, "a connection to the sign-in listener failed"); @@ -161,7 +206,11 @@ async fn receive_redirect(listener: &TcpListener, state: &str) -> Result) { +async fn answer_redirect( + mut stream: TcpStream, + (state, provider): (String, String), + found: mpsc::Sender, +) { let read = tokio::time::timeout( Duration::from_secs(REDIRECT_READ_TIMEOUT_SECS), request_target(&mut stream), @@ -173,7 +222,7 @@ async fn answer_redirect(mut stream: TcpStream, state: String, found: mpsc::Send let returned = redirect_state(&target); let ours = returned.as_deref() == Some(state.as_str()); let unstated_error = returned.is_none() && target.contains("error="); - if let (true, Err(error)) = (unstated_error, redirect_code(&target, &state)) { + if let (true, Err(error)) = (unstated_error, redirect_code(&target, (&state, &provider))) { say(&format!( "An error came back without the sign-in's state, so SQLLoad keeps waiting: {error}. If it came from your sign-in, press Ctrl-C and fix that." )); @@ -184,6 +233,7 @@ async fn answer_redirect(mut stream: TcpStream, state: String, found: mpsc::Send (true, true) => ("200 OK", SIGN_IN_FAILED_PAGE), (true, false) => ("200 OK", SIGNED_IN_PAGE), }; + let page = page.replace(PROVIDER_PLACEHOLDER, &provider); let response = format!( "HTTP/1.1 {status}\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{page}", page.len() @@ -249,6 +299,6 @@ fn open_browser(url: &str) -> bool { } /// Tell the person signing in what to do, on stderr so stdout stays the command's output. -fn say(text: &str) { +pub(crate) fn say(text: &str) { let _ = writeln!(std::io::stderr(), "{text}"); } diff --git a/crates/sqlload/src/auth/_helpers/token_cache.rs b/crates/sqlload/src/auth/_helpers/token_cache.rs index 4d1cbdaa..dbc8820e 100644 --- a/crates/sqlload/src/auth/_helpers/token_cache.rs +++ b/crates/sqlload/src/auth/_helpers/token_cache.rs @@ -1,13 +1,12 @@ -//! The file browser sign-ins are cached in, readable only by its owner, keyed by account, user, OAuth client and role. +//! The files browser sign-ins are cached in, readable only by their owner: Snowflake's keyed by account, user, OAuth client and role, Google's by OAuth client and connection. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; -use crate::auth::constants::TOKEN_CACHE_FILE; use crate::auth::models::{CachedTokens, OauthClient, OauthTokens, Secret}; -/// The token cache file: below `XDG_CACHE_HOME`, else `~/.cache` (`%LOCALAPPDATA%` on Windows). -pub(crate) fn token_cache_file() -> Option { +/// The token cache file `name`: below `XDG_CACHE_HOME`, else `~/.cache` (`%LOCALAPPDATA%` on Windows). +pub(crate) fn token_cache_file(name: &str) -> Option { let variable = |name: &str| { std::env::var_os(name) .filter(|value| !value.is_empty()) @@ -19,7 +18,7 @@ pub(crate) fn token_cache_file() -> Option { variable("XDG_CACHE_HOME").or_else(|| variable("HOME").map(|home| home.join(".cache"))) } }; - folder.map(|folder| folder.join(TOKEN_CACHE_FILE)) + folder.map(|folder| folder.join(name)) } /// The key one sign-in is cached under. @@ -39,7 +38,17 @@ pub(crate) fn cache_key(client: &OauthClient) -> String { /// The cached sign-in for `client`; none when the file is missing or unreadable. pub(crate) fn read_cached(client: &OauthClient) -> Option { - let cached = read_file(&client.cache_file).remove(&cache_key(client))?; + read_entry(&client.cache_file, &cache_key(client)) +} + +/// Cache `tokens` for `client`, keeping other sign-ins in the file. +pub(crate) fn write_cached(client: &OauthClient, tokens: &OauthTokens) -> Result<(), String> { + write_entry(&client.cache_file, &cache_key(client), tokens) +} + +/// The sign-in cached in `path` under `key`; none when the file is missing or unreadable. +pub(crate) fn read_entry(path: &Path, key: &str) -> Option { + let cached = read_file(path).remove(key)?; Some(OauthTokens { access_token: Secret(cached.access_token), renew_after: cached.renew_after, @@ -48,12 +57,11 @@ pub(crate) fn read_cached(client: &OauthClient) -> Option { }) } -/// Cache `tokens` for `client`, keeping other sign-ins in the file: written to a new file only its owner can read, then renamed over the old one. -pub(crate) fn write_cached(client: &OauthClient, tokens: &OauthTokens) -> Result<(), String> { - let path = &client.cache_file; +/// Cache `tokens` in `path` under `key`, keeping other sign-ins in the file: written to a new file only its owner can read, then renamed over the old one. +pub(crate) fn write_entry(path: &Path, key: &str, tokens: &OauthTokens) -> Result<(), String> { let mut all = read_file(path); all.insert( - cache_key(client), + key.to_owned(), CachedTokens { access_token: tokens.access_token.expose().to_owned(), renew_after: tokens.renew_after, @@ -89,12 +97,12 @@ fn read_file(path: &Path) -> BTreeMap { Ok(text) => text, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return BTreeMap::new(), Err(error) => { - tracing::warn!(%error, file = %path.display(), "cannot read the Snowflake sign-in cache"); + tracing::warn!(%error, file = %path.display(), "cannot read the sign-in cache"); return BTreeMap::new(); } }; serde_json::from_str(&text).unwrap_or_else(|error| { - tracing::warn!(%error, file = %path.display(), "the Snowflake sign-in cache is not valid; signing in again"); + tracing::warn!(%error, file = %path.display(), "the sign-in cache is not valid; signing in again"); BTreeMap::new() }) } diff --git a/crates/sqlload/src/auth/constants.rs b/crates/sqlload/src/auth/constants.rs index 4f76e0e2..8fbe2042 100644 --- a/crates/sqlload/src/auth/constants.rs +++ b/crates/sqlload/src/auth/constants.rs @@ -12,8 +12,12 @@ pub const OAUTH_REFRESH_MARGIN_SECS: u64 = 60; pub const DEFAULT_ACCESS_LIFETIME_SECS: u64 = 600; /// The token cache, below the user's cache folder. pub const TOKEN_CACHE_FILE: &str = "sqlload/snowflake-tokens.json"; +/// The Google sign-in cache, below the user's cache folder. +pub const GOOGLE_TOKEN_CACHE_FILE: &str = "sqlload/google-tokens.json"; +/// The provider of Google sign-ins, as messages name it. +pub const GOOGLE_PROVIDER: &str = "Google"; /// What the browser shows when Snowflake sent back an error instead of a code. -pub const SIGN_IN_FAILED_PAGE: &str = "SQLLoad

The Snowflake sign-in failed. Return to the terminal for the reason.

"; +pub const SIGN_IN_FAILED_PAGE: &str = "SQLLoad

The {provider} sign-in failed. Return to the terminal for the reason.

"; /// Where the browser sign-in redirects to: SQLLoad listens there. pub const LOOPBACK_HOST: &str = "127.0.0.1"; pub const LOOPBACK_SCHEME: &str = "http"; @@ -25,8 +29,22 @@ pub const REDIRECT_READ_TIMEOUT_SECS: u64 = 10; pub const BROWSER_CHECKS: u32 = 20; pub const BROWSER_CHECK_MILLIS: u64 = 100; /// What a request to the sign-in listener that is not this sign-in's redirect gets. -pub const NOT_THIS_SIGN_IN_PAGE: &str = "SQLLoad

This is not the Snowflake sign-in SQLLoad is waiting for.

"; +pub const NOT_THIS_SIGN_IN_PAGE: &str = "SQLLoad

This is not the {provider} sign-in SQLLoad is waiting for.

"; /// The most of a redirect request SQLLoad reads. pub const REDIRECT_REQUEST_LIMIT: usize = 16 * 1024; /// What the browser shows once the sign-in reached SQLLoad. -pub const SIGNED_IN_PAGE: &str = "SQLLoad

Signed in to Snowflake. You can close this tab and return to the terminal.

"; +pub const SIGNED_IN_PAGE: &str = "SQLLoad

Signed in to {provider}. You can close this tab and return to the terminal.

"; +/// The name in the pages above. +pub const PROVIDER_PLACEHOLDER: &str = "{provider}"; +/// The provider of Snowflake sign-ins, as messages name it. +pub const SNOWFLAKE_PROVIDER: &str = "Snowflake"; +/// Google's OAuth authorization endpoint, where a person consents in the browser. +pub const GOOGLE_AUTHORIZE_URL: &str = "https://accounts.google.com/o/oauth2/v2/auth"; +/// Google's OAuth token endpoint. +pub const GOOGLE_TOKEN_URL: &str = "https://oauth2.googleapis.com/token"; +/// The error Google's token endpoint answers a revoked, expired or unknown refresh token or code with. +pub const INVALID_GRANT: &str = "invalid_grant"; +/// How long Google's access tokens last when the answer doesn't say, in seconds. +pub const DEFAULT_GOOGLE_TOKEN_SECS: u64 = 3600; +/// The most of a Google token endpoint's error text an error shows. +pub const GOOGLE_ERROR_CHARS: usize = 300; diff --git a/crates/sqlload/src/auth/main/google_token_cache_file.rs b/crates/sqlload/src/auth/main/google_token_cache_file.rs new file mode 100644 index 00000000..02931447 --- /dev/null +++ b/crates/sqlload/src/auth/main/google_token_cache_file.rs @@ -0,0 +1,11 @@ +//! Where Google sign-ins are cached. + +use std::path::PathBuf; + +use crate::auth::_helpers::token_cache::token_cache_file; +use crate::auth::constants::GOOGLE_TOKEN_CACHE_FILE; + +/// The Google sign-in cache file, below the user's cache folder; `None` when there is none. +pub fn google_token_cache_file() -> Option { + token_cache_file(GOOGLE_TOKEN_CACHE_FILE) +} diff --git a/crates/sqlload/src/auth/main/google_user_token.rs b/crates/sqlload/src/auth/main/google_user_token.rs new file mode 100644 index 00000000..4e768bb0 --- /dev/null +++ b/crates/sqlload/src/auth/main/google_user_token.rs @@ -0,0 +1,16 @@ +//! A Google user's access token, renewed, or from a consent in the browser when a terminal is there. + +use crate::auth::_helpers::google_oauth::google_access_token; +use crate::auth::_helpers::sign_in::terminal_io; +use crate::auth::models::{GoogleUser, Secret}; +use crate::http::errors::ApiError; + +/// The access token to send as `user` now, other than `refused`: the one `held` while it has time left, else one renewed with the connection's or the cached refresh token, else one from a consent in the browser when a terminal is there to give it. A refresh token Google refuses is an error naming why and how to get a new one. +pub async fn google_user_token( + http: &reqwest::Client, + user: &GoogleUser, + held: &tokio::sync::Mutex>, + refused: Option<&Secret>, +) -> Result { + google_access_token(http, user, held, (refused, &terminal_io())).await +} diff --git a/crates/sqlload/src/auth/main/mod.rs b/crates/sqlload/src/auth/main/mod.rs index 6b2f1e01..c409546d 100644 --- a/crates/sqlload/src/auth/main/mod.rs +++ b/crates/sqlload/src/auth/main/mod.rs @@ -1,2 +1,4 @@ pub mod account_url; pub mod allow_sign_in; +pub mod google_token_cache_file; +pub mod google_user_token; diff --git a/crates/sqlload/src/auth/models.rs b/crates/sqlload/src/auth/models.rs index 26399cd8..62d58c43 100644 --- a/crates/sqlload/src/auth/models.rs +++ b/crates/sqlload/src/auth/models.rs @@ -7,7 +7,7 @@ use crate::auth::_helpers::sign_in::terminal_io; use crate::auth::_helpers::token_cache::token_cache_file; use crate::auth::constants::{ JWT_LIFETIME_SECS, JWT_REFRESH_MARGIN_SECS, KEYPAIR_JWT, LOCAL_APPLICATION, OAUTH, - PROGRAMMATIC_ACCESS_TOKEN, + PROGRAMMATIC_ACCESS_TOKEN, TOKEN_CACHE_FILE, }; use crate::auth::main::account_url::account_url; use crate::config::models::{AuthConfig, DestinationConfig}; @@ -63,6 +63,22 @@ pub struct OauthClient { pub http: reqwest::Client, } +/// A Google user who consents in the browser to an installed app's OAuth client: the client, a refresh token the connection gives (for unattended runs) or the one cached at consent, and Google's endpoints. +#[derive(Debug, Clone)] +pub struct GoogleUser { + pub client_id: String, + pub client_secret: Secret, + /// The connection's `refresh_token`; when unset, the one cached under `cache_key`, or a new consent's. + pub refresh_token: Option, + /// The one scope consent asks for. + pub scope: String, + /// The connection, as messages and the cache name it. + pub connection: String, + pub authorize_url: String, + pub token_url: String, + pub cache_file: std::path::PathBuf, +} + /// A browser sign-in's tokens in one run, and why signing in failed, after which the run asks nobody again. #[derive(Debug, Default)] pub struct OauthState { @@ -145,7 +161,7 @@ impl Authenticator { redirect_uri: redirect_uri.clone(), user: destination.user.clone(), role: destination.role.clone(), - cache_file: token_cache_file().ok_or_else(|| { + cache_file: token_cache_file(TOKEN_CACHE_FILE).ok_or_else(|| { ApiError::Auth( "no folder to cache the Snowflake sign-in in: set XDG_CACHE_HOME or HOME" .to_owned(), diff --git a/crates/sqlload/src/auth/tests.rs b/crates/sqlload/src/auth/tests.rs index 7e5dfc35..6c5e0796 100644 --- a/crates/sqlload/src/auth/tests.rs +++ b/crates/sqlload/src/auth/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_google_oauth.rs"] +mod google_oauth; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_oauth.rs"] diff --git a/crates/sqlload/src/auth/tests/helpers.rs b/crates/sqlload/src/auth/tests/helpers.rs index edff4da3..17b08998 100644 --- a/crates/sqlload/src/auth/tests/helpers.rs +++ b/crates/sqlload/src/auth/tests/helpers.rs @@ -7,17 +7,26 @@ use axum::extract::State; use axum::http::{StatusCode, Uri, header}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; +use serde_json::json; +use crate::auth::_helpers::google_oauth::{cache_key, google_access_token}; use crate::auth::_helpers::oauth::{challenge, fresh_access_token, now_secs}; use crate::auth::_helpers::sign_in::SignInIo; -use crate::auth::_helpers::token_cache::{read_cached, write_cached}; -use crate::auth::models::{OauthClient, OauthState, OauthTokens, Secret}; -use crate::auth::tests::test_types::{FakeOauth, LifecycleOutcome, TokenLifecycleTestCase}; +use crate::auth::_helpers::token_cache::{read_cached, read_entry, write_cached, write_entry}; +use crate::auth::models::{GoogleUser, OauthClient, OauthState, OauthTokens, Secret}; +use crate::auth::tests::test_types::{ + FakeGoogle, FakeOauth, LifecycleOutcome, TokenLifecycleTestCase, UserTokenOutcome, + UserTokenTestCase, +}; use crate::http::main::http_client::build_http_client; use crate::http::models::ConnectionCounter; +use crate::loader::errors::LoaderError; type Shared = Arc>; +/// The scope the Google tests ask for. +const GOOGLE_TEST_SCOPE: &str = "https://www.googleapis.com/auth/spreadsheets.readonly"; + pub(crate) fn block_on( body: impl std::future::Future>, ) -> Result { @@ -245,3 +254,179 @@ fn query_pairs(text: &str) -> BTreeMap { .map(|url| url.query_pairs().into_owned().collect()) .unwrap_or_default() } + +type SharedGoogle = Arc>; + +/// Ask for an access token as `test_case` sets things up, against a fresh fake Google and cache, then ask again. +pub(crate) async fn user_token_outcome( + test_case: &UserTokenTestCase, +) -> Result { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let fake: SharedGoogle = Arc::new(Mutex::new(FakeGoogle { + revoked: test_case + .revoked + .iter() + .map(|token| (*token).to_owned()) + .collect(), + ..FakeGoogle::default() + })); + let base = start_fake_google(fake.clone()).await?; + let user = GoogleUser { + client_id: "client.apps.googleusercontent.com".to_owned(), + client_secret: Secret(test_case.client_secret.to_owned()), + refresh_token: test_case.configured.map(|token| Secret(token.to_owned())), + scope: GOOGLE_TEST_SCOPE.to_owned(), + connection: "sheets".to_owned(), + authorize_url: format!("{base}/auth"), + token_url: format!("{base}/token"), + cache_file: folder.path().join("google-tokens.json"), + }; + let now = now_secs(); + test_case + .cached + .map(|(access, left, refresh)| { + let tokens = OauthTokens { + access_token: Secret(access.to_owned()), + renew_after: now.saturating_add_signed(left), + refresh_token: Some(Secret(refresh.to_owned())), + refresh_expires_at: None, + }; + write_entry(&user.cache_file, &cache_key(&user), &tokens) + }) + .transpose()?; + let refused: Option = test_case.refused.map(|token| Secret(token.to_owned())); + let held: tokio::sync::Mutex> = + tokio::sync::Mutex::new(refused.clone().map(|token| (token, i64::MAX))); + let http = + build_http_client(&ConnectionCounter::default()).map_err(|error| error.to_string())?; + let io = fake_browser(test_case.terminal, http.clone()); + let first = tokio::time::timeout( + std::time::Duration::from_secs(5), + google_access_token(&http, &user, &held, (refused.as_ref(), &io)), + ) + .await + .map_err(|_| "no token within 5 s".to_owned())?; + let retryable = first.as_ref().err().is_some_and(|error| { + error.is_retryable() && LoaderError::from(error.clone()).is_retryable() + }); + let outcome = first.map_or_else(|error| error.to_string(), |token| token.expose().to_owned()); + let grants = fake + .lock() + .map(|fake| fake.grants.clone()) + .map_err(|error| error.to_string())?; + let cached_refresh = read_entry(&user.cache_file, &cache_key(&user)) + .and_then(|cached| cached.refresh_token) + .map(|token| token.expose().to_owned()); + let repeated = google_access_token(&http, &user, &held, (None, &io)) + .await + .map_or_else(|error| error.to_string(), |token| token.expose().to_owned()); + let requests = fake + .lock() + .map(|fake| fake.grants.len()) + .map_err(|error| error.to_string())?; + Ok(UserTokenOutcome { + outcome, + retryable, + grants, + cached_refresh, + repeated: (repeated, requests), + }) +} + +/// A terminal that is there or not, with a browser that follows the consent page to its redirect; nothing is ever pasted. +fn fake_browser(terminal: bool, http: reqwest::Client) -> SignInIo { + SignInIo { + terminal: Box::new(move || terminal), + open_browser: Box::new(move |url| { + let (http, url) = (http.clone(), url.to_owned()); + tokio::spawn(async move { http.get(url).send().await }); + true + }), + read_line: Box::new(|| Err(std::io::Error::other("nothing is pasted"))), + } +} + +/// Serve the fake's consent page and token endpoint on a free local port. +async fn start_fake_google(state: SharedGoogle) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .map_err(|error| error.to_string())?; + let address = listener.local_addr().map_err(|error| error.to_string())?; + let router = Router::new() + .route("/auth", get(google_consent)) + .route("/token", post(google_token)) + .with_state(state); + tokio::spawn(async move { axum::serve(listener, router).await }); + Ok(format!("http://{address}")) +} + +/// Record the PKCE challenge and redirect back with a code, when the consent asks for offline read-only access to spreadsheets with the consent screen shown; else with an error. +async fn google_consent(State(state): State, uri: Uri) -> Response { + let query = query_pairs(uri.query().unwrap_or_default()); + let Ok(mut fake) = state.lock() else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let field = |name: &str| query.get(name).cloned().unwrap_or_default(); + fake.challenge = field("code_challenge"); + let asked = field("scope") == GOOGLE_TEST_SCOPE + && field("access_type") == "offline" + && field("prompt") == "consent select_account" + && field("code_challenge_method") == "S256"; + let answer = ["error=invalid_request", "code=fake-code"][usize::from(asked)]; + let location = format!( + "{}?{answer}&state={}", + field("redirect_uri"), + field("state") + ); + (StatusCode::FOUND, [(header::LOCATION, location)]).into_response() +} + +/// Answer a token request as Google does: `invalid_client` for a wrong secret, 503 for the refresh token `unavailable`, `invalid_grant` for a revoked refresh token or a code whose verifier does not match, else an access token (and a refresh token for a code). +async fn google_token(State(state): State, body: String) -> Response { + let form = query_pairs(&body); + let Ok(mut fake) = state.lock() else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let field = |name: &str| form.get(name).cloned().unwrap_or_default(); + fake.grants.push(field("grant_type")); + fake.issued += 1; + let issued = fake.issued; + let refresh = field("refresh_token"); + let exchanged = form.contains_key("code"); + let verified = + field("code") == "fake-code" && challenge(&field("code_verifier")) == fake.challenge; + let answers: [(bool, StatusCode, String); 5] = [ + ( + field("client_secret") != "secret", + StatusCode::UNAUTHORIZED, + json!({"error": "invalid_client", "error_description": "The OAuth client was not found."}).to_string(), + ), + ( + refresh == "unavailable", + StatusCode::SERVICE_UNAVAILABLE, + json!({"error": "backend_error", "error_description": "try later"}).to_string(), + ), + ( + refresh == "broken", + StatusCode::INTERNAL_SERVER_ERROR, + format!("{}", "x".repeat(400)), + ), + ( + fake.revoked.contains(&refresh) || (exchanged && !verified), + StatusCode::BAD_REQUEST, + json!({"error": "invalid_grant", "error_description": "Token has been expired or revoked."}).to_string(), + ), + ( + true, + StatusCode::OK, + json!({"access_token": format!("access-{issued}"), "expires_in": 3599, "token_type": "Bearer", "scope": GOOGLE_TEST_SCOPE, "refresh_token": exchanged.then(|| format!("refresh-{issued}"))}).to_string(), + ), + ]; + answers + .into_iter() + .find(|(applies, _, _)| *applies) + .map_or_else( + || StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |(_, status, body)| (status, body).into_response(), + ) +} diff --git a/crates/sqlload/src/auth/tests/test_google_oauth.rs b/crates/sqlload/src/auth/tests/test_google_oauth.rs new file mode 100644 index 00000000..01d5ea3d --- /dev/null +++ b/crates/sqlload/src/auth/tests/test_google_oauth.rs @@ -0,0 +1,210 @@ +use crate::auth::tests::helpers::{block_on, user_token_outcome}; +use crate::auth::tests::test_types::UserTokenTestCase; + +const NO_TERMINAL: &str = "no usable Google sign-in is cached for the Google account for connection \"sheets\", and signing in in the browser needs a terminal; run `sqlload plan` from a terminal once to sign in, or set the connection's refresh_token for unattended runs"; + +#[test] +fn given_a_google_user_connection_when_a_token_is_needed_then_it_is_renewed_consented_or_refused_clearly() +-> Result<(), String> { + let test_cases = [ + UserTokenTestCase { + description: "the connection's refresh token renews an access token, which the next read reuses; it is not written to the cache", + configured: Some("refresh-own"), + cached: None, + refused: None, + revoked: &[], + client_secret: "secret", + terminal: false, + expected_outcome: "access-1", + expected_grants: &["refresh_token"], + expected_cached_refresh: None, + expected_retryable: false, + expected_repeated: ("access-1", 1), + }, + UserTokenTestCase { + description: "a revoked refresh token in the connection fails, saying why and how to get a new one, without asking anyone to consent", + configured: Some("refresh-own"), + cached: None, + refused: None, + revoked: &["refresh-own"], + client_secret: "secret", + terminal: true, + expected_outcome: "Google refused the refresh_token of connection \"sheets\" (invalid_grant: Token has been expired or revoked.): it was revoked, has expired", + expected_grants: &["refresh_token"], + expected_cached_refresh: None, + expected_retryable: false, + expected_repeated: ("Google refused the refresh_token", 2), + }, + UserTokenTestCase { + description: "with nothing cached and no terminal, the run fails at once, saying how to sign in", + configured: None, + cached: None, + refused: None, + revoked: &[], + client_secret: "secret", + terminal: false, + expected_outcome: NO_TERMINAL, + expected_grants: &[], + expected_cached_refresh: None, + expected_retryable: false, + expected_repeated: (NO_TERMINAL, 0), + }, + UserTokenTestCase { + description: "with nothing cached and a terminal, the person consents in the browser for offline read-only access; the code is exchanged with its PKCE verifier and the refresh token cached", + configured: None, + cached: None, + refused: None, + revoked: &[], + client_secret: "secret", + terminal: true, + expected_outcome: "access-1", + expected_grants: &["authorization_code"], + expected_cached_refresh: Some("refresh-1"), + expected_retryable: false, + expected_repeated: ("access-1", 1), + }, + UserTokenTestCase { + description: "a cached access token with time left is used as it is", + configured: None, + cached: Some(("access-cached", 600, "refresh-cached")), + refused: None, + revoked: &[], + client_secret: "secret", + terminal: false, + expected_outcome: "access-cached", + expected_grants: &[], + expected_cached_refresh: Some("refresh-cached"), + expected_retryable: false, + expected_repeated: ("access-cached", 0), + }, + UserTokenTestCase { + description: "an expired cached access token is renewed with the cached refresh token, which is kept, as Google sends no new one", + configured: None, + cached: Some(("access-old", -10, "refresh-cached")), + refused: None, + revoked: &[], + client_secret: "secret", + terminal: false, + expected_outcome: "access-1", + expected_grants: &["refresh_token"], + expected_cached_refresh: Some("refresh-cached"), + expected_retryable: false, + expected_repeated: ("access-1", 1), + }, + UserTokenTestCase { + description: "an access token the Sheets API refused is replaced even though it has time left", + configured: None, + cached: Some(("access-cached", 600, "refresh-cached")), + refused: Some("access-cached"), + revoked: &[], + client_secret: "secret", + terminal: false, + expected_outcome: "access-1", + expected_grants: &["refresh_token"], + expected_cached_refresh: Some("refresh-cached"), + expected_retryable: false, + expected_repeated: ("access-1", 1), + }, + UserTokenTestCase { + description: "a revoked cached refresh token, with a terminal, leads to a new consent whose refresh token replaces it", + configured: None, + cached: Some(("access-old", -10, "refresh-cached")), + refused: None, + revoked: &["refresh-cached"], + client_secret: "secret", + terminal: true, + expected_outcome: "access-2", + expected_grants: &["refresh_token", "authorization_code"], + expected_cached_refresh: Some("refresh-2"), + expected_retryable: false, + expected_repeated: ("access-2", 2), + }, + UserTokenTestCase { + description: "a revoked cached refresh token, without a terminal, fails saying it was refused and how to sign in again", + configured: None, + cached: Some(("access-old", -10, "refresh-cached")), + refused: None, + revoked: &["refresh-cached"], + client_secret: "secret", + terminal: false, + expected_outcome: "Google refused the cached refresh token of connection \"sheets\" (invalid_grant: Token has been expired or revoked.), so it needs a new sign-in: no usable Google sign-in", + expected_grants: &["refresh_token"], + expected_cached_refresh: Some("refresh-cached"), + expected_retryable: false, + expected_repeated: ("Google refused the cached refresh token", 2), + }, + UserTokenTestCase { + description: "a wrong client secret fails, naming the client, and is not retried", + configured: Some("refresh-own"), + cached: None, + refused: None, + revoked: &[], + client_secret: "wrong", + terminal: true, + expected_outcome: "Google's token endpoint refused OAuth client client.apps.googleusercontent.com (invalid_client: The OAuth client was not found.); check the connection's client_id and client_secret", + expected_grants: &["refresh_token"], + expected_cached_refresh: None, + expected_retryable: false, + expected_repeated: ("refused OAuth client", 2), + }, + UserTokenTestCase { + description: "a token endpoint that is down fails as a rerun may fix", + configured: Some("unavailable"), + cached: None, + refused: None, + revoked: &[], + client_secret: "secret", + terminal: false, + expected_outcome: "HTTP 503 : Google's token endpoint: backend_error: try later", + expected_grants: &["refresh_token"], + expected_cached_refresh: None, + expected_retryable: true, + expected_repeated: ("HTTP 503", 2), + }, + UserTokenTestCase { + description: "a token endpoint answering 500 with an HTML page fails as a rerun may fix, the page cut short before its end", + configured: Some("broken"), + cached: None, + refused: None, + revoked: &[], + client_secret: "secret", + terminal: false, + expected_outcome: "HTTP 500 : Google's token endpoint: 500: xxx", + expected_grants: &["refresh_token"], + expected_cached_refresh: None, + expected_retryable: true, + expected_repeated: ("HTTP 500", 2), + }, + ]; + for test_case in test_cases { + let seen = block_on(user_token_outcome(&test_case))?; + assert!( + seen.outcome.contains(test_case.expected_outcome) && !seen.outcome.contains(""), + "{}: {}", + test_case.description, + seen.outcome + ); + assert_eq!( + (seen.grants, seen.cached_refresh.as_deref(), seen.retryable), + ( + test_case + .expected_grants + .iter() + .map(|grant| (*grant).to_owned()) + .collect::>(), + test_case.expected_cached_refresh, + test_case.expected_retryable + ), + "{}", + test_case.description + ); + assert!( + seen.repeated.0.contains(test_case.expected_repeated.0) + && seen.repeated.1 == test_case.expected_repeated.1, + "{}: asked again: {:?}", + test_case.description, + seen.repeated + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/auth/tests/test_oauth.rs b/crates/sqlload/src/auth/tests/test_oauth.rs index b0821429..e61cdbda 100644 --- a/crates/sqlload/src/auth/tests/test_oauth.rs +++ b/crates/sqlload/src/auth/tests/test_oauth.rs @@ -55,8 +55,8 @@ fn given_where_a_sign_in_ended_when_reading_its_code_then_only_a_code_for_this_r }, ]; for test_case in &test_cases { - let found = - redirect_code(test_case.redirected, "s1").unwrap_or_else(|error| error.to_string()); + let found = redirect_code(test_case.redirected, ("s1", "Snowflake")) + .unwrap_or_else(|error| error.to_string()); assert!( found.contains(test_case.expected_outcome), "{}: {found}", diff --git a/crates/sqlload/src/auth/tests/test_types.rs b/crates/sqlload/src/auth/tests/test_types.rs index b9cb4e00..0b296419 100644 --- a/crates/sqlload/src/auth/tests/test_types.rs +++ b/crates/sqlload/src/auth/tests/test_types.rs @@ -86,3 +86,51 @@ pub(crate) struct ChallengeTestCase { pub(crate) verifier: &'static str, pub(crate) expected_challenge: &'static str, } + +/// An access token a Google user's client needs, with what the connection, the cache, the fake Google and the terminal hold. +pub(crate) struct UserTokenTestCase { + pub(crate) description: &'static str, + /// The connection's own refresh token. + pub(crate) configured: Option<&'static str>, + /// The cached access token, its seconds until renewal, and its refresh token. + pub(crate) cached: Option<(&'static str, i64, &'static str)>, + /// The access token the server just refused, which the client holds as unexpired. + pub(crate) refused: Option<&'static str>, + /// Refresh tokens Google refuses as revoked; `unavailable` makes its token endpoint fail with 503, and `broken` with a 500 and an HTML page. + pub(crate) revoked: &'static [&'static str], + /// The client secret sent; the fake takes only `secret`. + pub(crate) client_secret: &'static str, + /// Whether a terminal is there to consent from, with a browser that follows the consent. + pub(crate) terminal: bool, + /// The token, or text the error contains. + pub(crate) expected_outcome: &'static str, + /// The grant type of every token request, in order. + pub(crate) expected_grants: &'static [&'static str], + /// The refresh token cached afterwards. + pub(crate) expected_cached_refresh: Option<&'static str>, + /// Whether the error is one a rerun may fix. + pub(crate) expected_retryable: bool, + /// What asking again gives (the token, or text the error contains), and the token requests made by then. + pub(crate) expected_repeated: (&'static str, usize), +} + +/// What the fake Google OAuth server was asked, and how it answers. +#[derive(Debug, Default)] +pub(crate) struct FakeGoogle { + pub(crate) grants: Vec, + /// The last consent's PKCE challenge, which a code exchange's verifier must match. + pub(crate) challenge: String, + pub(crate) revoked: Vec, + pub(crate) issued: u32, +} + +/// What one token request came to. +pub(crate) struct UserTokenOutcome { + /// The token, or the error. + pub(crate) outcome: String, + pub(crate) retryable: bool, + pub(crate) grants: Vec, + pub(crate) cached_refresh: Option, + /// What asking again gave, and how many token requests had been made by then. + pub(crate) repeated: (String, usize), +} diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index b45cf884..cba32f95 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -7,6 +7,9 @@ use crate::adapters::dispatch::main::dialect::files_destination::files_destinati use crate::adapters::dispatch::main::session::live_columns::live_columns; use crate::adapters::dispatch::main::session::text_values::text_values; use crate::adapters::dispatch::models::Destination; +use crate::auth::constants::{GOOGLE_AUTHORIZE_URL, GOOGLE_TOKEN_URL}; +use crate::auth::main::google_token_cache_file::google_token_cache_file; +use crate::auth::models::GoogleUser; use crate::bigquery_source::main::bigquery_reader::bigquery_reader; use crate::bigquery_source::main::read_catalog::read_catalog as read_bigquery_catalog; use crate::bigquery_source::main::read_rows::read_rows as read_bigquery_rows; @@ -59,7 +62,7 @@ use crate::project::models::{ TableSettings, WriteStrategy, }; use crate::schema_plan::models::{ColumnOrigin, DesiredColumn, LiveColumn}; -use crate::sheets_source::constants::{EMPTY_SHEET_COLUMN_KIND, SHEET_COLUMN_KIND}; +use crate::sheets_source::constants::{EMPTY_SHEET_COLUMN_KIND, SCOPE, SHEET_COLUMN_KIND}; use crate::sheets_source::main::duplicate_key::duplicate_key; use crate::sheets_source::main::read_grids::read_grids; use crate::sheets_source::main::sheet_batch::sheet_batch; @@ -67,6 +70,7 @@ use crate::sheets_source::main::sheet_table::sheet_table; use crate::sheets_source::main::sheets_client::sheets_client; use crate::sheets_source::main::tab_name::tab_name; use crate::sheets_source::main::tab_ranges::tab_ranges; +use crate::sheets_source::main::user_sheets_client::user_sheets_client; use crate::sheets_source::models::{Grid, HeaderRule, KeySource, SheetsClient}; use crate::similar_names::main::close_name::close_name; use crate::similar_names::main::nearest_name::nearest_name; @@ -374,7 +378,7 @@ pub(crate) fn bigquery_source_reader( Ok(bigquery_reader(connection, source.project.as_deref())?) } -/// A Sheets client for a `google_sheets` source's connection, with the key it names (or `GOOGLE_APPLICATION_CREDENTIALS`'s), and the spreadsheet it reads. +/// A Sheets client for a `google_sheets` source's connection, with the key it names (or `GOOGLE_APPLICATION_CREDENTIALS`'s) or as the Google user who consents to its OAuth client, and the spreadsheet it reads. pub(crate) fn sheets_reader( project: &ProjectConfig, source: &SourceConfig, @@ -383,7 +387,11 @@ pub(crate) fn sheets_reader( service_account_file, service_account_json, service_account_json_base64, + client_id, + client_secret, + refresh_token, base_url, + token_url, }) = project.connections.get(&source.connection) else { return Err(LoaderError::Invalid(format!( @@ -395,25 +403,60 @@ pub(crate) fn sheets_reader( service_account_json.is_some(), service_account_json_base64.is_some(), service_account_file.is_some(), + client_id.is_some(), ]; if given.into_iter().filter(|given| *given).count() > 1 { return Err(LoaderError::Invalid(format!( - "connection {:?} sets more than one of service_account_json, service_account_json_base64 and service_account_file; keep one", + "connection {:?} sets more than one of service_account_json, service_account_json_base64, service_account_file and client_id; keep one", source.connection ))); } - let key = match ( - service_account_json, - service_account_json_base64, - service_account_file, - ) { - (Some(json), _, _) => KeySource::Json(json.clone()), - (None, Some(encoded), _) => KeySource::Base64(encoded.clone()), - (None, None, Some(path)) => KeySource::File(path.clone()), - (None, None, None) => KeySource::Environment, - }; let http = build_http_client(&ConnectionCounter::default())?; - let client = sheets_client(http, &key, base_url.as_deref())?; + let client = match (client_id, client_secret) { + (Some(client_id), Some(client_secret)) => { + let cache_file = google_token_cache_file().ok_or_else(|| { + LoaderError::Invalid( + "no cache folder for the Google sign-in: set HOME or XDG_CACHE_HOME (LOCALAPPDATA on Windows)".to_owned(), + ) + })?; + let user = GoogleUser { + client_id: client_id.clone(), + client_secret: client_secret.clone(), + refresh_token: refresh_token.clone(), + scope: SCOPE.to_owned(), + connection: source.connection.clone(), + authorize_url: GOOGLE_AUTHORIZE_URL.to_owned(), + token_url: token_url.as_deref().unwrap_or(GOOGLE_TOKEN_URL).to_owned(), + cache_file, + }; + user_sheets_client(http, user, base_url.as_deref()) + } + (Some(_), None) => { + return Err(LoaderError::Invalid(format!( + "connection {:?} sets client_id without client_secret; set both, from the OAuth client's JSON file", + source.connection + ))); + } + (None, _) if client_secret.is_some() || refresh_token.is_some() => { + return Err(LoaderError::Invalid(format!( + "connection {:?} sets client_secret or refresh_token without client_id; set client_id to read as a Google user", + source.connection + ))); + } + (None, _) => { + let key = match ( + service_account_json, + service_account_json_base64, + service_account_file, + ) { + (Some(json), _, _) => KeySource::Json(json.clone()), + (None, Some(encoded), _) => KeySource::Base64(encoded.clone()), + (None, None, Some(path)) => KeySource::File(path.clone()), + (None, None, None) => KeySource::Environment, + }; + sheets_client(http, &key, base_url.as_deref())? + } + }; let spreadsheet_id = source.spreadsheet_id.clone().ok_or_else(|| { LoaderError::Invalid("a google_sheets source needs a spreadsheet_id".to_owned()) })?; diff --git a/crates/sqlload/src/project/models.rs b/crates/sqlload/src/project/models.rs index 586d2c3a..1913c31a 100644 --- a/crates/sqlload/src/project/models.rs +++ b/crates/sqlload/src/project/models.rs @@ -229,7 +229,7 @@ pub enum ConnectionConfig { #[serde(default)] password: Option, }, - /// Google Sheets, read with a service-account key: at most one of the three key settings, or the key file `GOOGLE_APPLICATION_CREDENTIALS` names. + /// Google Sheets, read with a service-account key (at most one of the three key settings, or the key file `GOOGLE_APPLICATION_CREDENTIALS` names), or as a Google user who consents to an installed app's OAuth client (`client_id` and `client_secret`). GoogleSheets { /// A path to the key file. #[serde(default)] @@ -240,9 +240,20 @@ pub enum ConnectionConfig { /// The key file's JSON text, base64-encoded. #[serde(default)] service_account_json_base64: Option, + /// An installed app's OAuth client, to read as the Google user who consents to it. + #[serde(default)] + client_id: Option, + #[serde(default)] + client_secret: Option, + /// The user's refresh token, for unattended runs; without it, the one cached at consent. + #[serde(default)] + refresh_token: Option, /// Overrides the Sheets API's root, for tests. #[serde(default)] base_url: Option, + /// Overrides Google's OAuth token endpoint, for tests. + #[serde(default)] + token_url: Option, }, /// An HTTP API, for Python sources; its settings reach table functions as `ctx.connection`. Http { diff --git a/crates/sqlload/src/sheets_source/_helpers/requests.rs b/crates/sqlload/src/sheets_source/_helpers/requests.rs index d353d043..cab65abf 100644 --- a/crates/sqlload/src/sheets_source/_helpers/requests.rs +++ b/crates/sqlload/src/sheets_source/_helpers/requests.rs @@ -1,5 +1,6 @@ //! Reads of a spreadsheet: retried, with a new token when the server refuses one, and the parts of their answers. +use crate::auth::main::google_user_token::google_user_token; use crate::bigquery_api::main::google_answer::google_answer; use crate::bigquery_api::main::google_token::google_token; use crate::http::errors::ApiError; @@ -7,7 +8,7 @@ use crate::http::models::RetryPolicy; use crate::sheets_source::constants::{ FORBIDDEN, GRID_SHEET_TYPE, NOT_FOUND, RETRY_FIRST_DELAY, SCOPE, UNAUTHORIZED, }; -use crate::sheets_source::models::SheetsClient; +use crate::sheets_source::models::{SheetsClient, SheetsCredential}; /// The spreadsheet's title in a read's answer. pub(crate) fn spreadsheet_title(answer: &serde_json::Value) -> String { @@ -80,13 +81,14 @@ async fn send( ) -> Result { let mut refused = None; loop { - let token = google_token( - &client.http, - &client.key, - &client.token, - (SCOPE, refused.as_ref()), - ) - .await?; + let token = match &client.credential { + SheetsCredential::ServiceAccount(key) => { + google_token(&client.http, key, &client.token, (SCOPE, refused.as_ref())).await? + } + SheetsCredential::User(user) => { + google_user_token(&client.http, user, &client.token, refused.as_ref()).await? + } + }; let response = client .http .get(url.clone()) diff --git a/crates/sqlload/src/sheets_source/constants.rs b/crates/sqlload/src/sheets_source/constants.rs index 1b141352..193f7cf0 100644 --- a/crates/sqlload/src/sheets_source/constants.rs +++ b/crates/sqlload/src/sheets_source/constants.rs @@ -51,3 +51,6 @@ pub const NOT_FOUND: u16 = 404; /// The HTTP status of a request whose token the server refused. pub const UNAUTHORIZED: u16 = 401; + +/// Who a client signed in as a Google user reads as, as messages name it. +pub const GOOGLE_USER: &str = "the Google account that signed in"; diff --git a/crates/sqlload/src/sheets_source/main/mod.rs b/crates/sqlload/src/sheets_source/main/mod.rs index 5c0486ec..b13f4c16 100644 --- a/crates/sqlload/src/sheets_source/main/mod.rs +++ b/crates/sqlload/src/sheets_source/main/mod.rs @@ -5,3 +5,4 @@ pub mod sheet_table; pub mod sheets_client; pub mod tab_name; pub mod tab_ranges; +pub mod user_sheets_client; diff --git a/crates/sqlload/src/sheets_source/main/sheets_client.rs b/crates/sqlload/src/sheets_source/main/sheets_client.rs index b9b66b39..2a8f5409 100644 --- a/crates/sqlload/src/sheets_source/main/sheets_client.rs +++ b/crates/sqlload/src/sheets_source/main/sheets_client.rs @@ -4,7 +4,7 @@ use crate::bigquery_api::main::parse_key::parse_key; use crate::http::errors::ApiError; use crate::sheets_source::_helpers::keys::key_text; use crate::sheets_source::constants::{BASE_URL, SERVICE_ACCOUNT_TYPE}; -use crate::sheets_source::models::{KeySource, SheetsClient}; +use crate::sheets_source::models::{KeySource, SheetsClient, SheetsCredential}; /// A client reading as the service account of the key `source` gives, against `base_url` (the public API when `None`); nothing is sent until the first read. pub fn sheets_client( @@ -32,7 +32,7 @@ pub fn sheets_client( .unwrap_or(BASE_URL) .trim_end_matches('/') .to_owned(), - key: parse_key(&text)?, + credential: SheetsCredential::ServiceAccount(parse_key(&text)?), token: std::sync::Arc::default(), }) } diff --git a/crates/sqlload/src/sheets_source/main/user_sheets_client.rs b/crates/sqlload/src/sheets_source/main/user_sheets_client.rs new file mode 100644 index 00000000..079c7d19 --- /dev/null +++ b/crates/sqlload/src/sheets_source/main/user_sheets_client.rs @@ -0,0 +1,22 @@ +//! A Sheets API client reading as a Google user. + +use crate::auth::models::GoogleUser; +use crate::sheets_source::constants::BASE_URL; +use crate::sheets_source::models::{SheetsClient, SheetsCredential}; + +/// A client reading as the Google user who consents to `user`'s OAuth client, against `base_url` (the public API when `None`); nothing is sent, and nobody asked to sign in, until the first read. +pub fn user_sheets_client( + http: reqwest::Client, + user: GoogleUser, + base_url: Option<&str>, +) -> SheetsClient { + SheetsClient { + http, + base_url: base_url + .unwrap_or(BASE_URL) + .trim_end_matches('/') + .to_owned(), + credential: SheetsCredential::User(Box::new(user)), + token: std::sync::Arc::default(), + } +} diff --git a/crates/sqlload/src/sheets_source/models.rs b/crates/sqlload/src/sheets_source/models.rs index d57db53a..80bccd76 100644 --- a/crates/sqlload/src/sheets_source/models.rs +++ b/crates/sqlload/src/sheets_source/models.rs @@ -1,27 +1,38 @@ use std::sync::Arc; -use crate::auth::models::Secret; +use crate::auth::models::{GoogleUser, Secret}; use crate::bigquery_api::models::ServiceAccountKey; use crate::column_types::models::ColumnType; +use crate::sheets_source::constants::GOOGLE_USER; -/// What the Sheets API needs: the shared HTTP pool, the API root, the service-account key and the token it last gave. +/// What the Sheets API needs: the shared HTTP pool, the API root, who reads (a service account or a signed-in Google user) and the token it last gave. #[derive(Debug, Clone)] pub struct SheetsClient { pub(crate) http: reqwest::Client, /// The spreadsheets root, with no trailing `/`. pub(crate) base_url: String, - pub(crate) key: ServiceAccountKey, + pub(crate) credential: SheetsCredential, /// The current access token and when it expires, in milliseconds since the epoch. pub(crate) token: Arc>>, } impl SheetsClient { - /// The service account the client reads as. + /// Who the client reads as: the service account's address, or the Google user who signed in. pub fn account(&self) -> &str { - &self.key.client_email + match &self.credential { + SheetsCredential::ServiceAccount(key) => &key.client_email, + SheetsCredential::User(_) => GOOGLE_USER, + } } } +/// Who a client reads spreadsheets as. +#[derive(Debug, Clone)] +pub enum SheetsCredential { + ServiceAccount(ServiceAccountKey), + User(Box), +} + /// Where a connection's service-account key comes from. #[derive(Debug, Clone, PartialEq, Eq)] pub enum KeySource { diff --git a/crates/sqlload/tests/loader/sheets/helpers.rs b/crates/sqlload/tests/loader/sheets/helpers.rs index 5e05b58d..b187602c 100644 --- a/crates/sqlload/tests/loader/sheets/helpers.rs +++ b/crates/sqlload/tests/loader/sheets/helpers.rs @@ -164,3 +164,63 @@ header_row = 2 database = database.display(), ) } + +/// A project reading the fake spreadsheet's `Locations` tab as a Google user of an installed app's OAuth client, with the connection's `refresh` token if any, into DuckDB at `database`. +pub(crate) fn user_project_text( + (base_url, token_url): (&str, &str), + refresh: Option<&str>, + database: &std::path::Path, +) -> String { + let refresh = refresh.map_or_else(String::new, |token| format!("refresh_token = '{token}'\n")); + format!( + r#" +[connections.sheets] +kind = "google_sheets" +client_id = "client.apps.googleusercontent.com" +client_secret = "secret" +{refresh}base_url = '{base_url}' +token_url = '{token_url}' + +[connections.local] +kind = "duckdb" +database = '{database}' + +[targets.dev] +connection = "local" +loader_schema = "raw" + +[sources.reference] +kind = "google_sheets" +description = "Reference tables kept in a spreadsheet." +connection = "sheets" +spreadsheet_id = "fake-sheet" +ranges = {{ locations = "Locations" }} +"#, + database = database.display() + ) +} + +/// Run `sqlload` with `args` on `config`, its sign-in cache below `cache`: the exit code and the output. +pub(crate) fn run_with_cache( + config: &std::path::Path, + args: &[&str], + cache: &std::path::Path, +) -> Result<(Option, String), String> { + let output = std::process::Command::new(env!("CARGO_BIN_EXE_sqlload")) + .arg("--config") + .arg(config) + .arg("--no-color") + .args(args) + .env("XDG_CACHE_HOME", cache) + .env("LOCALAPPDATA", cache) + .output() + .map_err(|error| error.to_string())?; + Ok(( + output.status.code(), + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ), + )) +} diff --git a/crates/sqlload/tests/loader/test_sheets_to_duckdb.rs b/crates/sqlload/tests/loader/test_sheets_to_duckdb.rs index 79c0f5ec..77045086 100644 --- a/crates/sqlload/tests/loader/test_sheets_to_duckdb.rs +++ b/crates/sqlload/tests/loader/test_sheets_to_duckdb.rs @@ -1,8 +1,11 @@ use sqlload_testkit::fake_sheets::main::start_fake_sheets::start_fake_sheets; use crate::helpers::{duckdb_text, run_async, run_sqlload}; -use crate::sheets_helpers::{duplicated_tabs, edited_tabs, first_tabs, project_text, retyped_tabs}; -use crate::test_types::{SheetsLoadTestCase, SheetsRefusalTestCase}; +use crate::sheets_helpers::{ + duplicated_tabs, edited_tabs, first_tabs, project_text, retyped_tabs, run_with_cache, + user_project_text, +}; +use crate::test_types::{SheetsLoadTestCase, SheetsRefusalTestCase, SheetsUserTestCase}; const LOCATIONS: &str = "SELECT string_agg(Code || ':' || coalesce(Name, '') || ':' || coalesce(CAST(Opened AS VARCHAR), '') || ':' || coalesce(CAST(Active AS VARCHAR), '') || ':' || coalesce(CAST(Capacity AS VARCHAR), '') || ':' || coalesce(CAST(\"Updated At\" AS VARCHAR), '') || ':' || coalesce(Ref, ''), ',' ORDER BY Code) || '|' || (SELECT string_agg(column_name || ' ' || data_type, ',' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_schema = 'raw' AND table_name = 'reference__locations' AND column_name NOT LIKE '\\_sqlload%' ESCAPE '\\') FROM raw.reference__locations"; @@ -98,6 +101,16 @@ fn given_a_google_sheets_source_when_planning_then_a_shared_table_name_and_a_rep source: "[sources.keyed]\nkind = \"google_sheets\"\ndescription = \"Codes keyed by code.\"\nconnection = \"sheets\"\nspreadsheet_id = \"fake-sheet\"\nranges = { codes = \"Codes\" }\nunique_key = [\"code\"]\nwrite_strategy = \"merge\"\n", expected_output: "Codes!A2 and Codes!A4 hold the same unique_key (code)", }, + SheetsRefusalTestCase { + description: "a connection with both a service-account key and an OAuth client", + source: "[connections.mixed]\nkind = \"google_sheets\"\nservice_account_json = '{}'\nclient_id = \"c\"\nclient_secret = \"s\"\n[sources.mixed]\nkind = \"google_sheets\"\ndescription = \"Mixed.\"\nconnection = \"mixed\"\nspreadsheet_id = \"fake-sheet\"\n", + expected_output: "connection \"mixed\" sets more than one of service_account_json, service_account_json_base64, service_account_file and client_id; keep one", + }, + SheetsRefusalTestCase { + description: "an OAuth client without its secret", + source: "[connections.half]\nkind = \"google_sheets\"\nclient_id = \"c\"\n[sources.half]\nkind = \"google_sheets\"\ndescription = \"Half.\"\nconnection = \"half\"\nspreadsheet_id = \"fake-sheet\"\n", + expected_output: "connection \"half\" sets client_id without client_secret", + }, ]; fake.place("fake-sheet", duplicated_tabs()); for test_case in test_cases { @@ -113,3 +126,72 @@ fn given_a_google_sheets_source_when_planning_then_a_shared_table_name_and_a_rep Ok(()) }) } + +#[test] +fn given_a_google_user_connection_when_loading_then_its_refresh_token_reads_and_a_refused_one_fails_for_good() +-> Result<(), String> { + run_async(async { + let fake = start_fake_sheets().await?; + fake.place("fake-sheet", first_tabs()); + let test_cases = [ + SheetsUserTestCase { + description: "the connection's refresh token is exchanged for an access token that reads the sheet", + refresh: Some("refresh-own"), + revoked: &[], + expected_code: 0, + expected_output: "\"status\": \"loaded\"", + expected_grant: Some("refresh_token"), + }, + SheetsUserTestCase { + description: "a revoked refresh token fails with exit 1 and retryable false, saying how to get a new one", + refresh: Some("refresh-own"), + revoked: &["refresh-own"], + expected_code: 1, + expected_output: "Google refused the refresh_token of connection \\\"sheets\\\" (invalid_grant: Token has been expired or revoked.)", + expected_grant: Some("refresh_token"), + }, + SheetsUserTestCase { + description: "with no refresh token and nothing cached, a --json run asks nobody to sign in and fails at once, saying how to", + refresh: None, + revoked: &[], + expected_code: 1, + expected_output: "signing in in the browser needs a terminal; run `sqlload plan` from a terminal once to sign in", + expected_grant: None, + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let config = folder.path().join("sqlload.toml"); + let project = user_project_text( + (&fake.base_url, &fake.token_url), + test_case.refresh, + &folder.path().join("out.duckdb"), + ); + std::fs::write(&config, project).map_err(|error| error.to_string())?; + { + let mut state = fake.state.lock(); + state.grants.clear(); + state.revoked_refresh_tokens = test_case + .revoked + .iter() + .map(|token| (*token).to_owned()) + .collect(); + } + let (code, output) = run_with_cache(&config, &["load", "--json"], folder.path())?; + let grants = fake.state.lock().grants.clone(); + assert!( + code == Some(test_case.expected_code) + && output.contains(test_case.expected_output) + && output + .contains(&format!("\"retryable\": {}", test_case.expected_code == 75)) + && grants.is_empty() == test_case.expected_grant.is_none() + && grants + .iter() + .all(|grant| Some(grant.as_str()) == test_case.expected_grant), + "{}: exit {code:?}, grants {grants:?}:\n{output}", + test_case.description + ); + } + Ok(()) + }) +} diff --git a/crates/sqlload/tests/loader/test_types.rs b/crates/sqlload/tests/loader/test_types.rs index a3b6f186..1b8ecaef 100644 --- a/crates/sqlload/tests/loader/test_types.rs +++ b/crates/sqlload/tests/loader/test_types.rs @@ -908,3 +908,16 @@ pub(crate) struct SftpSnowflakePlanTestCase { pub(crate) expected_success: bool, pub(crate) expected_output: &'static str, } + +pub(crate) struct SheetsUserTestCase { + pub(crate) description: &'static str, + /// The connection's `refresh_token`. + pub(crate) refresh: Option<&'static str>, + /// Refresh tokens the fake refuses as revoked. + pub(crate) revoked: &'static [&'static str], + pub(crate) expected_code: i32, + /// Text the output holds. + pub(crate) expected_output: &'static str, + /// The grant type every token request has (plan and load each ask for a token), or none when no token is asked for. + pub(crate) expected_grant: Option<&'static str>, +} diff --git a/website/src/content/docs/docs/sources/google-sheets.mdx b/website/src/content/docs/docs/sources/google-sheets.mdx index dce49fb5..99be6ecd 100644 --- a/website/src/content/docs/docs/sources/google-sheets.mdx +++ b/website/src/content/docs/docs/sources/google-sheets.mdx @@ -1,6 +1,6 @@ --- title: "Google Sheets" -description: "Load a spreadsheet's tabs or ranges as tables, with typed dates, numbers and booleans, read with a service-account key." +description: "Load a spreadsheet's tabs or ranges as tables, with typed dates, numbers and booleans, read with a service-account key or as a signed-in Google user." --- From 0.27.0. A Google Sheets source loads tabs or ranges of one spreadsheet, one table each. Values arrive typed @@ -29,10 +29,61 @@ access to spreadsheets and nothing else. | `service_account_json` | The service-account key file's JSON text, normally `${ENV:...}`. | | `service_account_json_base64` | The same, base64-encoded, for secret stores that dislike newlines and quotes. | | `service_account_file` | A path to the key file. | +| `client_id`, `client_secret` | From 0.36.0 (not released yet). An OAuth client, to read as a Google user instead. See [Sign in as a Google user](#sign-in-as-a-google-user). | +| `refresh_token` | From 0.36.0 (not released yet). The Google user's refresh token, for unattended runs. | -Set at most one; `plan` refuses more. With none, SQLLoad reads the key file `GOOGLE_APPLICATION_CREDENTIALS` names. -`sqlload debug` shows which service account reads each source and how many tabs it sees; the key -itself never appears. Workload identity and user sign-in are not supported yet. +Set at most one of the three key settings and `client_id`; `plan` refuses more. With none, SQLLoad +reads the key file `GOOGLE_APPLICATION_CREDENTIALS` names. `sqlload debug` shows which service +account reads each source and how many tabs it sees; the key itself never appears. Workload identity +is not supported yet. + +## Sign in as a Google user + +From 0.36.0 (not released yet). +Status: user OAuth tested against a simulated Google endpoint, not yet against live Google. +The service-account route above is tested against live Google. + +When spreadsheets can't be shared with a service account, SQLLoad can read them as a Google user +who consents once in the browser. It asks only for the `spreadsheets.readonly` scope. + +1. In the Google Cloud console, enable the Google Sheets API. Then create an OAuth client of type + **Desktop app** (APIs & Services → Credentials). Its JSON file holds `client_id` and + `client_secret`. A desktop app's secret isn't confidential, but keep it out of the repository + anyway. +2. Set them on the connection: + + ```toml + [connections.sheets] + kind = "google_sheets" + client_id = "${ENV:GOOGLE_OAUTH_CLIENT_ID}" + client_secret = "${ENV:GOOGLE_OAUTH_CLIENT_SECRET}" + ``` + +3. Run `sqlload plan` from a terminal. SQLLoad opens the browser at Google's consent page and listens + on a local port for the redirect. On a machine without a browser, it prints the address: open it + elsewhere, then paste the address the browser ends on. SQLLoad caches the refresh token Google + sends in `sqlload/google-tokens.json` below your cache folder (`XDG_CACHE_HOME` or `~/.cache`, + or `%LOCALAPPDATA%` on Windows), readable only by you. Later runs on that machine renew access + tokens with it. + +For scheduled runs on another machine, put the refresh token in a secret and set +`refresh_token = "${ENV:GOOGLE_REFRESH_TOKEN}"`. SQLLoad then never asks anyone to sign in. + +A run that can't ask anyone to sign in fails at once and says how to: no terminal, `--json`, or +nothing cached. Google may refuse a refresh token: when it's revoked, when the account's password +changes, or after 7 days for a client whose consent screen is still in testing. The connection's +own `refresh_token` then fails the table and names the cause; get a new one by signing in from a +terminal with `refresh_token` unset. A refused cached token leads to a new consent when a terminal +is there. These failures exit 1, not 75, as running again unchanged won't fix them. A token endpoint +that is down or throttling is retried like any other Sheets request, and exits 75 if it stays down. + +Share the spreadsheet with the signed-in account, or sign in as an account that can open it. + +The cache file holds one entry per OAuth client and connection name, keyed `" "`, +shared by every project on the machine that uses them. Its `refresh_token` field is the token to copy +for unattended runs. To sign in as another account, delete that entry (or revoke SQLLoad's access at +[myaccount.google.com/permissions](https://myaccount.google.com/permissions)) and run `sqlload plan` +from a terminal again; Google's account chooser then lets you pick the account. ## Source