From 31cb0c838f9a89b3ccca41e1647deb5fc1acd681 Mon Sep 17 00:00:00 2001 From: Tahmid Ahmed Date: Tue, 3 Nov 2026 22:59:39 -0500 Subject: [PATCH 1/2] DINO-1164: add shared env file gated by sops using the main dev sops vault. renamed a couple envs to be a bit clearer --- .env.shared.yaml | 18 ++++++++ .gitignore | 1 - .sops.yaml | 6 +++ Justfile | 42 +++++++++++++++++-- example.env | 5 ++- internal/leetcode-auth-service/src/config.rs | 26 ++++++++---- .../src/grpc/client/lass.rs | 4 +- pre-commit | 18 ++++++++ 8 files changed, 103 insertions(+), 17 deletions(-) create mode 100644 .env.shared.yaml create mode 100644 .sops.yaml create mode 100644 pre-commit diff --git a/.env.shared.yaml b/.env.shared.yaml new file mode 100644 index 000000000..5ffe2545d --- /dev/null +++ b/.env.shared.yaml @@ -0,0 +1,18 @@ +#ENC[AES256_GCM,data:lwEUlgahzI6WrD1pWZpJvu7y/kHaSkvDtyhOgr+w,iv:4C68ZjeQrirDSA9JIjlNrw5aGn86I9lCxgHpwcMqqnY=,tag:oRPOcsl/hn7VTxuP1aH1Pg==,type:comment] +GRPC_LASS_HOST: ENC[AES256_GCM,data:hJF0BjX7oxgIz4b9BJlZ,iv:iydJ2BA9HNcKwwQa2F1zOLjWl1F9REe2HNfJXWbhXnQ=,tag:bC7JOxw1SIaqIsxu8owtfQ==,type:str] +GRPC_LASS_TLS: ENC[AES256_GCM,data:St1kCiY=,iv:jVY3T125sBNNDnUYU3p/BwJ06kwb+P2s+Kp8rADX/6M=,tag:leH+GwUMpbN2rnuie9DD7w==,type:bool] +#ENC[AES256_GCM,data:EvW6i5dIgQXGMK8iusUyxo2pi/IkUaeFKLQaSoNa+c1fIvZx08CbFYDK0qvdSHi2e4NCUUWSAeBWLwpY/StxXN8=,iv:eA/IeC9M1qzcirNVy6+A8ZXzaVNz7U/EYWug3u7qWc4=,tag:qO7PTtSv/G3ho5Xgn5OrLQ==,type:comment] +#ENC[AES256_GCM,data:U44BoakMMLL9cytDScShCXvEUNM+IgLwoN1hTlwVKV4wHsXnGO8Z3/6QNm5uloYlTrLBimlGdPAYSG2mkGWzt+danIT+ZYp4htTcwYEqavrRluLBHeYwTA==,iv:iLNM6pBFMCD+/witn9FNoF+S9vlUJ7KTO4y9wcBSx7U=,tag:kVJSpjcbAi01/tYFELn22g==,type:comment] +#ENC[AES256_GCM,data:cspOKG6FGXALsPyj5UgVkjboseY=,iv:0C+DYQPqtGEBDy7RWlr3TdbAVCmVHBLIkVJ5T1cOZjQ=,tag:yG1XW+r3F054N6XP+ihrVA==,type:comment] +GRPC_REFLECTION: ENC[AES256_GCM,data:wZkdRA==,iv:mHC/PpflafL2Y47JRGvd74hRHhTiSOCcZsIbIsJE/jc=,tag:ouTTWt1HBc7k2OV7nxFJPA==,type:bool] +sops: + azure_kv: + - created_at: "2026-11-04T03:51:14Z" + enc: LIqBmTBP_QHlPZRv3LaG55C539IVeJN2cx-WlPC-b1tprPrKLu29-L0EFDXKEuT8VbKzqVmfw7xGd1NnpkLn6MgGW7hjYAtlrE0ej-XzUQczrcWTEs_tDUtGsngfw6CZARMsoxfbXThe2VuwN6KQ0yItM5Ep-TyaCTJ4_nEBFjZPFSGPM_j3JGagrMdnsNOQAs1AtPgYAe0-s4BaOONUa8T-nA4qW6blx7ltbkpMnhw75fVcYbbg_ur7v6D0ZaqyBpN5C8RhZeDoIaixkX0fyQ2y9kJDkQCG-P505T2T-Ao9mhontNgZJoRdVBWc8059KB3E-pZmaZwAg2eFLdDSOQ + name: sops-key + vault_url: https://sops-master.vault.azure.net + version: 90784f2986de4514934b4d1f682d3e59 + lastmodified: "2026-11-04T03:51:17Z" + mac: ENC[AES256_GCM,data:pfm5MTtHrkLusl7QtyreZq3giH3YVgG3IV3FVwHVZaWOTOzdNssvDTUbavi2ysS48o4p0UViLNQsjgmCOx+r4kFvLsflYJ/kWn7PBhzNeP89KH8Tur+2rk4NHGoG5Bz+CQFXxgjv1jrtCmaBhgf3EDR8KXW2Xpac7Gu7rBGkAuI=,iv:olMBXxmNXEuI95W81POb/zE/1LPLfdNynH03TquB43o=,tag:2ipLrUjQm358OITm1IyH9A==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.3 diff --git a/.gitignore b/.gitignore index f3a1c8def..5dc9e1fe2 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,6 @@ node_modules .env # Local secrets. -.env.shared .env.ci .env.ci-app .env.staging diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 000000000..77cbfe524 --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,6 @@ +stores: + yaml: + indent: 2 +creation_rules: + - path_regex: .env.shared.yaml + azure_keyvault: "https://sops-master.vault.azure.net/keys/sops-key/" diff --git a/Justfile b/Justfile index 23f0b62f9..fa74274c3 100644 --- a/Justfile +++ b/Justfile @@ -1,5 +1,15 @@ set shell := ["bash", "-uc"] +### Azure cmds + +# logout and authenticate again +re-auth: + az logout && just auth + +# authenticate +auth: + just install-pre-scripts && az login + # Migrate to local DB migrate *args: dotenvx run -- bash ./mvnw flyway:migrate -Dflyway.locations=filesystem:./db {{args}} @@ -22,11 +32,11 @@ copy-stg *args: # Run the backend Spring server backend-dev *args: - dotenvx run -f .env -f .env.shared -- bash ./mvnw -Dspring-boot.run.profiles=dev spring-boot:run {{args}} + sops exec-env .env.shared.yaml 'dotenvx run --overload -f .env -- bash ./mvnw -Dspring-boot.run.profiles=dev spring-boot:run {{args}}' # Run the leetcode auth service las-dev *args: - cd ./internal/leetcode-auth-service && cargo build && dotenvx run -f ../../.env -- cargo run + cd ./internal/leetcode-auth-service && cargo build && sops exec-env ../../.env.shared.yaml 'dotenvx run --overload -f ../../.env -- cargo run {{args}}' # Run the leetcode auth stealer service lass-dev *args: @@ -87,9 +97,35 @@ email-dev *args: email-gen *args: cd email && bash email.sh {{args}} +# Use if you created a new secret file and need to encrypt it with SOPS +# If you are editing a file that has already been encrypted, see `just edit` +encrypt file *args: + just install-pre-scripts && sops --encrypt --in-place {{ file }} {{ args }} + +# Securely edit any secret file that is encrypted with SOPS +# You can change the editor it will call on by changing the $EDITOR environment variable +# +# you can choose to set it one time for the scope of the command: `EDITOR="nvim" just edit secrets.yaml` +# or you can put `export EDITOR=nvim` inside of your `~/.zshrc`, then restart your terminal & run: `just edit secrets.yaml` +# +# if you would like to use VSCode, `EDITOR="code --wait"` (You may have to follow this first: https://code.visualstudio.com/docs/setup/mac#_launch-vs-code-from-the-command-line) +edit file *args: + just install-pre-scripts && sops edit {{ file }} {{ args }} + # Run the dev servers (backend & frontend & auth service) dev *args: - npx mprocs "just backend-dev" "just frontend-dev" "just las-dev" "just lass-dev" {{args}} + just install-pre-scripts && npx mprocs "just backend-dev" "just frontend-dev" "just las-dev" "just lass-dev" {{args}} + +# Git hooks are installed on almost every command +# so that we don't accidentally add unencrypted secrets to the git history. +install-pre-scripts: + just install-pre-commit && just install-pre-push + +install-pre-commit: + cp pre-commit .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit + +install-pre-push: + cp pre-commit .git/hooks/pre-push && chmod +x .git/hooks/pre-push # Run the dev servers (backend & frontend & auth service) but the backend will launch a debugger server. devd *args: diff --git a/example.env b/example.env index e42597e77..f869170e0 100644 --- a/example.env +++ b/example.env @@ -80,8 +80,9 @@ SERVER_URL=http://localhost:5173 LEETCODE_AUTH_USERNAME= LEETCODE_AUTH_PASSWORD= REDIS_URL=redis://localhost:6379/0 -LASS_URL=http://localhost:50051 -LASS_TIMEOUT_SECONDS=180 +GPRC_LASS_HOST=localhost:50051 +GPRC_LASS_TLS=false +GRPC_LASS_TIMEOUT_SECONDS=180 REFRESH_LOCK_TTL_SECONDS=210 REFRESH_CHECK_INTERVAL_SECONDS=900 TOKEN_MAX_AGE_SECONDS=14400 diff --git a/internal/leetcode-auth-service/src/config.rs b/internal/leetcode-auth-service/src/config.rs index ae017a589..9797a1710 100644 --- a/internal/leetcode-auth-service/src/config.rs +++ b/internal/leetcode-auth-service/src/config.rs @@ -12,8 +12,8 @@ pub struct Config { /// Identifies this pod as the holder of the refresh lock. pub hostname: String, pub redis_url: String, - pub lass_url: String, - pub lass_timeout: Duration, + pub grpc_lass_url: String, + pub grpc_lass_timeout: Duration, /// Defaults to outlasting one LASS call, so the lock does not expire mid-refresh. pub refresh_lock_ttl: Duration, pub leetcode_auth_username: String, @@ -30,8 +30,8 @@ impl Config { grpc_reflection = self.grpc_reflection, version = self.version.as_deref().unwrap_or("N/A"), hostname = self.hostname, - lass_url = self.lass_url, - lass_timeout = ?self.lass_timeout, + lass_url = self.grpc_lass_url, + lass_timeout = ?self.grpc_lass_timeout, refresh_lock_ttl = ?self.refresh_lock_ttl, refresh_check_interval = ?self.refresh_check_interval, token_max_age = ?self.token_max_age, @@ -44,7 +44,7 @@ impl Config { } fn from_vars(var: impl Fn(&str) -> Option) -> anyhow::Result { - let lass_timeout = seconds(&var, "LASS_TIMEOUT_SECONDS", 180); + let lass_timeout = seconds(&var, "GRPC_LASS_TIMEOUT_SECONDS", 180); Ok(Self { grpc_reflection: var("GRPC_REFLECTION") @@ -61,10 +61,18 @@ impl Config { .filter(|v| !v.is_empty()) .unwrap_or_else(|| "unknown".to_string()), redis_url: required(&var, "REDIS_URL")?, - lass_url: var("LASS_URL") - .filter(|v| !v.is_empty()) - .unwrap_or_else(|| "http://localhost:50051".to_string()), - lass_timeout, + grpc_lass_url: { + let host = var("GRPC_LASS_HOST") + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| "localhost:50051".to_string()); + let tls = var("GRPC_LASS_TLS") + .map(|b| b.parse::().ok()) + .flatten() + .unwrap_or_else(|| false); + + format!("{}{host}", if tls { "http://" } else { "https://" }) + }, + grpc_lass_timeout: lass_timeout, refresh_lock_ttl: seconds( &var, "REFRESH_LOCK_TTL_SECONDS", diff --git a/internal/leetcode-auth-service/src/grpc/client/lass.rs b/internal/leetcode-auth-service/src/grpc/client/lass.rs index 20b8b9bc3..3de2db0bb 100644 --- a/internal/leetcode-auth-service/src/grpc/client/lass.rs +++ b/internal/leetcode-auth-service/src/grpc/client/lass.rs @@ -31,10 +31,10 @@ impl GrpcLASS { /// Connects lazily so this service starts, and keeps serving stored credentials, while LASS is /// down. pub fn new(config: &Config) -> anyhow::Result { - let channel = Endpoint::from_shared(config.lass_url.clone()) + let channel = Endpoint::from_shared(config.grpc_lass_url.clone()) .context("LASS_URL must be a valid URI")? .connect_timeout(Duration::from_secs(5)) - .timeout(config.lass_timeout) + .timeout(config.grpc_lass_timeout) .connect_lazy(); Ok(Self { diff --git a/pre-commit b/pre-commit new file mode 100644 index 000000000..df492c7c7 --- /dev/null +++ b/pre-commit @@ -0,0 +1,18 @@ +#!/bin/bash +FILES=$(git diff --cached --name-only | grep -E '\.env\.shared\.yaml$') +EXIT_CODE=0 +for FILE in $FILES; do + if [ -f "$FILE" ]; then + if ! grep -q "sops:" "$FILE"; then + echo "--------------------------------------------------------" + echo "ERROR: Unencrypted secret file detected: $FILE" + echo "Please encrypt this file by running \`just e $FILE\`," + echo "then run \`git add $FILE\`." + echo "--------------------------------------------------------" + EXIT_CODE=1 + fi + fi +done +exit $EXIT_CODE +EOF +chmod +x .git/hooks/pre-commit From 31cc6821795edf098562084503db03c199d453c4 Mon Sep 17 00:00:00 2001 From: Tahmid Ahmed Date: Thu, 5 Nov 2026 00:39:14 -0500 Subject: [PATCH 2/2] DINO-1164: pr fixes --- Justfile | 5 +++-- pre-commit | 21 +++++++++------------ pre-push | 29 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 14 deletions(-) mode change 100644 => 100755 pre-commit create mode 100755 pre-push diff --git a/Justfile b/Justfile index fa74274c3..b8a0bf52f 100644 --- a/Justfile +++ b/Justfile @@ -121,11 +121,12 @@ dev *args: install-pre-scripts: just install-pre-commit && just install-pre-push +# `--git-path` resolves the shared hooks dir, so this also works inside a worktree. install-pre-commit: - cp pre-commit .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit + cp pre-commit "$(git rev-parse --git-path hooks)/pre-commit" && chmod +x "$(git rev-parse --git-path hooks)/pre-commit" install-pre-push: - cp pre-commit .git/hooks/pre-push && chmod +x .git/hooks/pre-push + cp pre-push "$(git rev-parse --git-path hooks)/pre-push" && chmod +x "$(git rev-parse --git-path hooks)/pre-push" # Run the dev servers (backend & frontend & auth service) but the backend will launch a debugger server. devd *args: diff --git a/pre-commit b/pre-commit old mode 100644 new mode 100755 index df492c7c7..b216339a7 --- a/pre-commit +++ b/pre-commit @@ -1,18 +1,15 @@ #!/bin/bash -FILES=$(git diff --cached --name-only | grep -E '\.env\.shared\.yaml$') +# Checks the staged version of each shared secret file, which is what gets committed. +FILES=$(git diff --cached --name-only --diff-filter=ACMR | grep -E '\.env\.shared\.yaml$') EXIT_CODE=0 for FILE in $FILES; do - if [ -f "$FILE" ]; then - if ! grep -q "sops:" "$FILE"; then - echo "--------------------------------------------------------" - echo "ERROR: Unencrypted secret file detected: $FILE" - echo "Please encrypt this file by running \`just e $FILE\`," - echo "then run \`git add $FILE\`." - echo "--------------------------------------------------------" - EXIT_CODE=1 - fi + if ! git show ":$FILE" | grep -q "sops:"; then + echo "--------------------------------------------------------" + echo "ERROR: Unencrypted secret file staged: $FILE" + echo "Please encrypt this file by running \`just encrypt $FILE\`," + echo "then run \`git add $FILE\`." + echo "--------------------------------------------------------" + EXIT_CODE=1 fi done exit $EXIT_CODE -EOF -chmod +x .git/hooks/pre-commit diff --git a/pre-push b/pre-push new file mode 100755 index 000000000..bac15ede2 --- /dev/null +++ b/pre-push @@ -0,0 +1,29 @@ +#!/bin/bash +# Checks every outgoing commit, so a plaintext secret committed before the hooks were installed +# cannot be pushed. Git passes " " lines on stdin. +ZERO=0000000000000000000000000000000000000000 +EXIT_CODE=0 +while read -r LOCAL_REF LOCAL_SHA REMOTE_REF REMOTE_SHA; do + # Deleting a remote branch pushes no commits. + [ "$LOCAL_SHA" = "$ZERO" ] && continue + + if [ "$REMOTE_SHA" = "$ZERO" ]; then + COMMITS=$(git rev-list "$LOCAL_SHA" --not --remotes) + else + COMMITS=$(git rev-list "$REMOTE_SHA..$LOCAL_SHA") + fi + + for COMMIT in $COMMITS; do + FILES=$(git diff-tree --no-commit-id --name-only -r --diff-filter=ACMR "$COMMIT" | grep -E '\.env\.shared\.yaml$') + for FILE in $FILES; do + if ! git show "$COMMIT:$FILE" | grep -q "sops:"; then + echo "--------------------------------------------------------" + echo "ERROR: Unencrypted secret file in commit $(git rev-parse --short "$COMMIT"): $FILE" + echo "Encrypt it with \`just encrypt $FILE\` and rewrite that commit before pushing." + echo "--------------------------------------------------------" + EXIT_CODE=1 + fi + done + done +done +exit $EXIT_CODE