Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -674,6 +674,31 @@ describe('app security review command boundary', () => {
})
})

test('--sarif leaves out the findings that shopify.app-security.json suppresses, from both sources', async () => {
await inTemporaryDirectory(async (directory) => {
const appDirectory = await createApp(directory, {deterministic: deterministicFindings, agent: agentFindings})
const withoutFile = await runCommand(['--path', directory, '--sarif'])
// Without the file, the agent finding of CREDENTIAL_LOG_LEAKAGE and the deterministic finding of EOL_API_VERSION
// are results.
expect(
JSON.parse(withoutFile.stdout).runs[0].results.map((sarifResult: {ruleId: string}) => sarifResult.ruleId),
).toStrictEqual(['CREDENTIAL_LOG_LEAKAGE', 'EOL_API_VERSION'])
await writeAppSecuritySuppressionsFixture(appDirectory, suppressionEntries)

const result = await runCommand(['--path', directory, '--blocking', 'low', '--sarif'])

// The file's entries suppress both findings, so the log has no results, and every check keeps its rule.
expect(result.exitCode).toBe(0)
const [run] = JSON.parse(result.stdout).runs
expect(run.results).toStrictEqual([])
expect(run.tool.driver.rules.map((rule: {id: string}) => rule.id)).toStrictEqual([
'CREDENTIAL_LOG_LEAKAGE',
'MISSING_TENANT_ISOLATION',
'EOL_API_VERSION',
])
})
})

test('refuses --sarif with --json', async () => {
await inTemporaryDirectory(async (directory) => {
await createApp(directory, {deterministic: deterministicFindings})
Expand Down
61 changes: 60 additions & 1 deletion packages/app/src/cli/services/security-review/sarif.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ const suppressionsFilePath = onWindows
: '/apps/my-app/shopify.app-security.json'
const bundlePath = onWindows ? 'C:\\apps\\my-app\\public\\bundle.js' : '/apps/my-app/public/bundle.js'
const bundleUri = onWindows ? 'file:///C:/apps/my-app/public/bundle.js' : 'file:///apps/my-app/public/bundle.js'
const configurationPath = onWindows ? 'C:\\apps\\my-app\\shopify.app.toml' : '/apps/my-app/shopify.app.toml'
const productPath = onWindows
? 'C:\\apps\\my-app\\app\\routes\\products.[id].tsx'
: '/apps/my-app/app/routes/products.[id].tsx'
const scope = {includeDirs: [], excludes: [], noGitIgnore: false}
const sourceResult = {version: 2, status: 'executed', reason: null, analysisMode: null, generatedAt: null} as const

Expand Down Expand Up @@ -159,6 +163,27 @@ const notApplicableCheck = {
},
} satisfies AppSecurityReviewResult['checks'][number]

/** A deterministic finding whose message and evidence have brackets. */
const redirectCheck = {
...passedCheck,
id: 'INSECURE_WEBHOOK_URL',
title: 'Configured callback URL is not HTTPS',
severity: 'high',
findings: [
{
source: 'deterministic',
number: 5,
disposition: 'active',
location: {filePath: configurationPath, line: 4, column: 1},
message: 'OAuth redirect URI at auth.redirect_urls[0] is not a conservative secure URL.',
snippet: null,
evidence: [{location: {filePath: productPath, line: 3, column: null}, quote: String.raw`path.match(/\[id\]/)`}],
fix: null,
suppression: null,
},
],
} satisfies AppSecurityReviewResult['checks'][number]

const resultsFiles = {
deterministic: {
path: `${resultsDirectory}${onWindows ? '\\' : '/'}deterministic-findings.json`,
Expand Down Expand Up @@ -245,6 +270,25 @@ const noResultsFile = appSecurityReviewJsonOutputSchema.validate({
resultsDigest: null,
})

/** A finding, its evidence and a coverage gap whose messages have brackets. */
const bracketed = appSecurityReviewJsonOutputSchema.validate({
...result,
coverage: {
filesScanned: 1,
filesSkipped: [],
gaps: [
{
code: 'skipped-file',
message: 'app/routes/products.[id].tsx was too large to scan.',
checkId: null,
filePath: productPath,
},
],
scanDirectories: [],
},
checks: [redirectCheck],
})

function missingCheckNotification(checkId: string) {
return {
level: 'warning',
Expand All @@ -260,6 +304,7 @@ describe('appSecurityReviewSarif', () => {
['a run narrowed by --check-id', incomplete, ['OPEN_REDIRECT', 'CREDENTIAL_LOG_LEAKAGE']],
['checks without results', withoutResults, []],
['no results file', noResultsFile, []],
['messages with brackets', bracketed, []],
])('writes a valid SARIF 2.1.0 log for %s', async (_description, value, checkIds) => {
const validation = jsonSchemaValidate(
appSecurityReviewSarif(value, {checkIds}),
Expand Down Expand Up @@ -330,7 +375,9 @@ describe('appSecurityReviewSarif', () => {
ruleId: 'CREDENTIAL_LOG_LEAKAGE',
ruleIndex: 0,
level: 'error',
message: {text: 'The session access token is written to the server log.'},
message: {
text: 'The session access token is written to the server log. Evidence: [app/routes/orders.tsx:10](0), [app/routes/orders.tsx](1).',
},
locations: [
{
physicalLocation: {
Expand Down Expand Up @@ -363,6 +410,18 @@ describe('appSecurityReviewSarif', () => {
})
})

test('escapes brackets and backslashes in messages, since SARIF reads [text](id) as a link to a location', () => {
const [run] = appSecurityReviewSarif(bracketed, {checkIds: []}).runs

expect(run!.results[0]!.message.text).toBe(
String.raw`OAuth redirect URI at auth.redirect_urls\[0\] is not a conservative secure URL. Evidence: [app/routes/products.\[id\].tsx:3](0).`,
)
expect(run!.results[0]!.relatedLocations![0]!.message).toStrictEqual({text: String.raw`path.match(/\\\[id\\\]/)`})
expect(run!.invocations[0]!.toolExecutionNotifications.map((notification) => notification.message)).toStrictEqual([
{text: String.raw`app/routes/products.\[id\].tsx was too large to scan.`},
])
})

test('lists unresolved, not applicable and missing checks and coverage gaps, and marks the execution unsuccessful', () => {
const [run] = appSecurityReviewSarif(incomplete, {checkIds: []}).runs

Expand Down
52 changes: 43 additions & 9 deletions packages/app/src/cli/services/security-review/sarif.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {CLI_KIT_VERSION} from '@shopify/cli-kit/common/version'
import {pathToFileURL} from '@shopify/cli-kit/node/fs'
import {relativePath} from '@shopify/cli-kit/node/path'
import type {AppSecurityReviewResult} from './types.js'

/**
Expand Down Expand Up @@ -91,7 +92,9 @@ export function narrowedSarifCategory(checkIds: string[], clientId: string | nul
/**
* One rule per reviewed check, and one result per active finding. A superseded finding is left out, since the check's
* agent result replaced it, as in the review. A suppressed finding is left out too: GitHub code scanning ignores SARIF
* `suppressions` and would open an alert for it, so its alert, if open, closes as fixed instead.
* `suppressions` and would open an alert for it, so its alert, if open, closes as fixed instead. A finding's evidence is
* a related location of its result, which the result's message links to, as code scanning shows only the related
* locations that the message links to.
*
* The invocation lists each unresolved or not applicable check, each check that neither results file has, and each
* coverage gap, as a notification. Code scanning reads a rule without results as passing, so a check without results
Expand Down Expand Up @@ -135,16 +138,29 @@ export function appSecurityReviewSarif(result: AppSecurityReviewResult, {checkId
results: result.checks.flatMap((check, ruleIndex) =>
check.findings
.filter((finding) => finding.disposition === 'active')
.map((finding) => sarifResult(check, ruleIndex, finding)),
.map((finding) => sarifResult(check, ruleIndex, finding, result.app.directory)),
),
},
],
}
}

/**
* A plain text message. SARIF reads `[text](id)` in a message as a link to a location of the result, so literal
* brackets, such as those of `redirect_urls[0]` or of a route file's `[id]`, and the backslash that escapes them, are
* escaped with a backslash.
*/
function sarifMessage(text: string): SarifMessage {
return {text: escapeSarifText(text)}
}

function escapeSarifText(text: string): string {
return text.replace(/[\\[\]]/g, '\\$&')
}

const NO_RESULTS_FILE_NOTIFICATION: SarifNotification = {
level: 'error',
message: {text: 'No results file to review: run shopify app security check first.'},
message: sarifMessage('No results file to review: run shopify app security check first.'),
descriptor: {id: 'no-results-file'},
}

Expand All @@ -163,7 +179,7 @@ function checkNotifications(check: AppSecurityCheck, ruleIndex: number): SarifNo
return [
{
level: check.status === 'unresolved' ? 'warning' : 'note',
message: {text: `${check.id} ${STATUS_TEXT[check.status]}${because}`},
message: sarifMessage(`${check.id} ${STATUS_TEXT[check.status]}${because}`),
descriptor: {id: check.status},
associatedRule: {id: check.id, index: ruleIndex},
},
Expand All @@ -177,7 +193,7 @@ function checkNotifications(check: AppSecurityCheck, ruleIndex: number): SarifNo
function missingCheckNotification(checkId: string): SarifNotification {
return {
level: 'warning',
message: {text: `${checkId} has no result: neither results file has this check.`},
message: sarifMessage(`${checkId} has no result: neither results file has this check.`),
descriptor: {id: 'no-result'},
}
}
Expand All @@ -194,7 +210,7 @@ function gapNotifications(gap: AppSecurityGap, checks: AppSecurityCheck[]): Sari
return [
{
level: 'warning',
message: {text: gap.message},
message: sarifMessage(gap.message),
descriptor: {id: gap.code},
...(gap.checkId === null ? {} : {associatedRule: {id: gap.checkId, index: ruleIndex}}),
...(gap.filePath === null ? {} : {locations: [{physicalLocation: {artifactLocation: fileUri(gap.filePath)}}]}),
Expand All @@ -215,17 +231,30 @@ function sarifRule(check: AppSecurityCheck): SarifRule {
}
}

function sarifResult(check: AppSecurityCheck, ruleIndex: number, finding: AppSecurityFinding): SarifResult {
/**
* The message ends with a link to each evidence location, with the location relative to the app as its text, as in the
* review.
*/
function sarifResult(
check: AppSecurityCheck,
ruleIndex: number,
finding: AppSecurityFinding,
appDirectory: string,
): SarifResult {
const relatedLocations = finding.evidence.map((evidence, id) => ({
id,
physicalLocation: sarifPhysicalLocation(evidence.location),
...(evidence.quote === null ? {} : {message: {text: evidence.quote}}),
...(evidence.quote === null ? {} : {message: sarifMessage(evidence.quote)}),
}))
const text = escapeSarifText(finding.message === '' ? check.title : finding.message)
const evidenceLinks = finding.evidence.map(
(evidence, id) => `[${escapeSarifText(appRelativeLocation(evidence.location, appDirectory))}](${id})`,
)
return {
ruleId: check.id,
ruleIndex,
level: LEVELS[check.severity],
message: {text: finding.message === '' ? check.title : finding.message},
message: {text: evidenceLinks.length === 0 ? text : `${text} Evidence: ${evidenceLinks.join(', ')}.`},
locations: [{physicalLocation: sarifPhysicalLocation(finding.location, finding.snippet)}],
...(relatedLocations.length > 0 ? {relatedLocations} : {}),
properties: {
Expand All @@ -235,6 +264,11 @@ function sarifResult(check: AppSecurityCheck, ruleIndex: number, finding: AppSec
}
}

function appRelativeLocation(location: AppSecurityLocation, appDirectory: string): string {
const path = relativePath(appDirectory, location.filePath)
return location.line === null ? path : `${path}:${location.line}`
}

/**
* An absolute `file:` URI, as SARIF consumers resolve them: GitHub's upload-sarif action makes it relative to the
* repository checkout. A location without a line covers the whole file, so it has no region, and the snippet, which
Expand Down
Loading