Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ diff <(sql-pipe --schema v1.csv) <(sql-pipe --schema v2.csv)
| `--stats` | **Deprecated.** Use `--inspect stats` instead. Load input and print per-column statistics (column name, type, non-null count, min, max, mean) as a formatted table. Mean is blank for non-numeric columns. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format`. Mutually exclusive with a query, `--columns`, `--validate`, `--sample`, and `--output`. |
| `--profile` | Alias for `--stats` |
| `--schema` | **Deprecated.** Use `--inspect schema` instead. Load input and print the inferred `CREATE TABLE` DDL to stdout. One DDL block per input file; stdin uses table `t`. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format`. Mutually exclusive with a query, `--columns`, `--validate`, `--sample`, `--stats`, `--output`, `--save`, and `-f`/`--file`. |
| `--inspect <mode>` | Inspect input data without running a query. `<mode>` is one of: `columns` (list column names, one per line; with `-v` shows inferred types), `validate` (parse entire input, print summary: `OK: <n> rows, <m> columns`), `sample` [`<n>`] (print schema comment block and first `<n>` rows as CSV, default 10), `stats` (per-column statistics: type, non-null count, min, max, mean), `schema` (print inferred CREATE TABLE DDL). Mutually exclusive with a query argument, `--output`, `--explain`, and other inspect modes. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, yaml, xml, parquet). `--sample` implies `--header`. `--sample` and `--stats` are mutually exclusive with `--json` output. |
| `--inspect <mode>` | Inspect input data without running a query. `<mode>` is one of: `columns` (list column names, one per line; with `-v` shows inferred types), `validate` (parse entire input, print summary: `OK: <n> rows, <m> columns`), `sample` [`<n>`] (print schema comment block and first `<n>` rows as CSV, default 10; `<n>` must be a positive integer placed directly after the mode — to sample a file whose name is a number, use `./N` or put it after `--`), `stats` (per-column statistics: type, non-null count, min, max, mean), `schema` (print inferred CREATE TABLE DDL). Mutually exclusive with a query argument, `--output`, `--explain`, and other inspect modes. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, yaml, xml, parquet). `--sample` implies `--header`. `--sample` and `--stats` are mutually exclusive with `--json` output. |
| `-S`, `--save <file>` | Use `<file>` as the SQLite database instead of an in-memory database. The file persists after sql-pipe exits and can be queried with `sqlite3`. Implies disk-backed behavior; cannot be combined with `--disk` or special modes (`--columns`, `--validate`, `--sample`, `--stats`, `--schema`, `--explain`). |
| `--disk` | Use a file-backed temporary database instead of `:memory:`. Enables processing datasets larger than available RAM. Cannot be combined with `--save`. |
| `--no-stdin` | Do not read from stdin. Prevents hangs in CI pipelines where stdin is a non-TTY pipe without EOF. |
Expand Down Expand Up @@ -579,7 +579,7 @@ $ sql-pipe --inspect schema data.csv
CREATE TABLE t (id INTEGER, name TEXT, age INTEGER);
```

All `--inspect` modes support `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, yaml, xml, parquet). `--inspect sample` implies `--header`. `--inspect sample` and `--inspect stats` are mutually exclusive with `--json` output.
All `--inspect` modes support `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, yaml, xml, parquet). `--inspect sample` implies `--header`. `--inspect sample` and `--inspect stats` are mutually exclusive with `--json` output. In sample mode, the optional count must be a positive integer placed directly after the mode (default: 10); to sample a file whose name is a number, use `./N` or put it after `--` (e.g. `sql-pipe --inspect sample -- 2`).

### Interactive REPL with --repl

Expand Down
125 changes: 125 additions & 0 deletions build.zig
Original file line number Diff line number Diff line change
Expand Up @@ -1095,6 +1095,131 @@ pub fn build(b: *std.Build) void {
test_sample_fewer_rows.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_sample_fewer_rows.step);

// ─── --inspect sample count regression tests (#237) ─────────────────────

// Integration test 237-1: --inspect sample 2 consumes the count (exact stdout, exit 0, schema on stderr, no deprecation warning)
const test_inspect_sample_count = b.addSystemCommand(&.{
"bash", "-c",
\\result=$(printf 'id,region,amount\n1,east,100\n2,west,200\n3,north,300\n4,south,400\n' \
\\ | ./zig-out/bin/sql-pipe --inspect sample 2 2>/dev/null) || exit 1
\\expected=$(printf 'id,region,amount\n1,east,100\n2,west,200')
\\[ "$result" = "$expected" ] || exit 1
\\stderr=$(printf 'id,region\n1,east\n' | ./zig-out/bin/sql-pipe --inspect sample 1 2>&1 >/dev/null)
\\echo "$stderr" | grep -q '^# Schema (2 columns):' || exit 1
\\! echo "$stderr" | grep -q 'deprecated' || exit 1
});
test_inspect_sample_count.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_inspect_sample_count.step);

// Integration test 237-2: --inspect=sample 2 consumes the count
const test_inspect_equals_sample_count = b.addSystemCommand(&.{
"bash", "-c",
\\result=$(printf 'id,region,amount\n1,east,100\n2,west,200\n3,north,300\n4,south,400\n' \
\\ | ./zig-out/bin/sql-pipe --inspect=sample 2 2>/dev/null) || exit 1
\\expected=$(printf 'id,region,amount\n1,east,100\n2,west,200')
\\[ "$result" = "$expected" ] || exit 1
});
test_inspect_equals_sample_count.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_inspect_equals_sample_count.step);

// Integration test 237-3: default count is 10 with a 12-row input, for both syntaxes
const test_inspect_sample_default_ten = b.addSystemCommand(&.{
"bash", "-c",
\\input=$(printf 'n\n'; i=1; while [ $i -le 12 ]; do echo "row$i"; i=$((i+1)); done)
\\expected=$(printf 'n\n'; i=1; while [ $i -le 10 ]; do echo "row$i"; i=$((i+1)); done)
\\for form in '--inspect sample' '--inspect=sample'; do
\\ result=$(printf '%s\n' "$input" | ./zig-out/bin/sql-pipe $form 2>/dev/null) || exit 1
\\ [ "$result" = "$expected" ] || exit 1
\\done
});
test_inspect_sample_default_ten.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_inspect_sample_default_ten.step);

// Integration test 237-4: the count is not turned into a file (#237)
const test_inspect_sample_count_not_file = b.addSystemCommand(&.{
"bash", "-c",
\\expected=$(head -3 tests/fixtures/orders.csv)
\\for form in '--inspect sample 2' '--inspect=sample 2'; do
\\ result=$(./zig-out/bin/sql-pipe $form --no-stdin tests/fixtures/orders.csv 2>/dev/null) || exit 1
\\ [ "$result" = "$expected" ] || exit 1
\\done
});
test_inspect_sample_count_not_file.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_inspect_sample_count_not_file.step);

// Integration test 237-5: flags and files immediately after the mode are not consumed as the count
const test_inspect_sample_flag_and_file_after_mode = b.addSystemCommand(&.{
"bash", "-c",
\\result=$(printf 'a\tb\n1\t2\n3\t4\n' | ./zig-out/bin/sql-pipe --inspect sample --tsv 2>/dev/null) || exit 1
\\expected=$(printf 'a\tb\n1\t2\n3\t4')
\\[ "$result" = "$expected" ] || exit 1
\\expected_file=$(head -11 tests/fixtures/orders.csv)
\\result=$(./zig-out/bin/sql-pipe --inspect sample --no-stdin tests/fixtures/orders.csv 2>/dev/null) || exit 1
\\[ "$result" = "$expected_file" ] || exit 1
});
test_inspect_sample_flag_and_file_after_mode.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_inspect_sample_flag_and_file_after_mode.step);

// Integration test 237-6: zero count is a usage error (exit exactly 1) for both syntaxes
const test_inspect_sample_zero = b.addSystemCommand(&.{
"bash", "-c",
\\for form in '--inspect sample 0' '--inspect=sample 0'; do
\\ result=$(printf 'a,b\n1,2\n' | ./zig-out/bin/sql-pipe $form 2>/dev/null)
\\ rc=$?
\\ [ "$rc" = 1 ] || exit 1
\\ [ -z "$result" ] || exit 1
\\done
\\msg=$(printf 'a,b\n1,2\n' | ./zig-out/bin/sql-pipe --inspect sample 0 2>&1 >/dev/null)
\\echo "$msg" | grep -q 'positive integer' || exit 1
});
test_inspect_sample_zero.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_inspect_sample_zero.step);

// Integration test 237-7: --inspect sample 2 --no-stdin with no source reports the missing source (exit 1), does not open '2'
const test_inspect_sample_count_no_source = b.addSystemCommand(&.{
"bash", "-c",
\\msg=$(printf 'a,b\n1,2\n' | ./zig-out/bin/sql-pipe --inspect sample 2 --no-stdin 2>&1 >/dev/null; echo "EXIT:$?")
\\echo "$msg" | grep -q -- '--inspect requires an input source when --no-stdin is set' || exit 1
\\echo "$msg" | grep -Fxq 'EXIT:1' || exit 1
});
test_inspect_sample_count_no_source.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_inspect_sample_count_no_source.step);

// Integration test 237-8: a file literally named '2' works via -- 2, ./2, and after --no-stdin
const test_inspect_sample_file_named_two = b.addSystemCommand(&.{
"bash", "-c",
\\tmp=$(mktemp -d) || exit 1
\\trap 'rm -rf "$tmp"' EXIT
\\printf 'x\n10\n20\n' > "$tmp/2"
\\bin="$PWD/zig-out/bin/sql-pipe"
\\expected=$(printf 'x\n10\n20')
\\result=$(cd "$tmp" && "$bin" --inspect sample --no-stdin -- 2 2>/dev/null) || exit 1
\\[ "$result" = "$expected" ] || exit 1
\\result=$(cd "$tmp" && "$bin" --inspect sample --no-stdin ./2 2>/dev/null) || exit 1
\\[ "$result" = "$expected" ] || exit 1
\\result=$(cd "$tmp" && "$bin" --inspect sample --no-stdin 2 2>/dev/null) || exit 1
\\[ "$result" = "$expected" ] || exit 1
\\result=$(cd "$tmp" && printf 'x\n10\n' | "$bin" --inspect columns --no-stdin 2 2>/dev/null) || exit 1
\\[ "$result" = "x" ] || exit 1
});
test_inspect_sample_file_named_two.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_inspect_sample_file_named_two.step);

// Integration test 237-9: --sample aliases keep working (count, warning); canonical form does not warn
const test_inspect_sample_aliases_compat = b.addSystemCommand(&.{
"bash", "-c",
\\result=$(printf 'a,b\n1,2\n3,4\n5,6\n' | ./zig-out/bin/sql-pipe --sample 2 2>/dev/null) || exit 1
\\expected=$(printf 'a,b\n1,2\n3,4')
\\[ "$result" = "$expected" ] || exit 1
\\result=$(printf 'a,b\n1,2\n3,4\n5,6\n' | ./zig-out/bin/sql-pipe --sample=2 2>/dev/null) || exit 1
\\[ "$result" = "$expected" ] || exit 1
\\msg=$(printf 'a,b\n1,2\n' | ./zig-out/bin/sql-pipe --sample 1 2>&1 >/dev/null)
\\echo "$msg" | grep -q 'warning: --sample is deprecated' || exit 1
});
test_inspect_sample_aliases_compat.step.dependOn(b.getInstallStep());
test_step.dependOn(&test_inspect_sample_aliases_compat.step);


// Integration test 95: 2-char delimiter (||) splits fields correctly
const test_delimiter_double_pipe = b.addSystemCommand(&.{
"bash", "-c",
Expand Down
4 changes: 3 additions & 1 deletion docs/sql-pipe.1.scd
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,9 @@ OPTIONS
Inspect input data without running a query. *<mode>* is one of:
*columns* (list column names, one per line; with *-v* shows inferred types),
*validate* (parse entire input, print summary: OK: <n> rows, <m> columns),
*sample* [<n>] (print schema comment block and first <n> rows as CSV, default 10),
*sample* [<n>] (print schema comment block and first <n> rows as CSV, default 10; <n> must
be a positive integer placed directly after the mode — to sample a file whose name is a
number, use ./N or put it after --),
*stats* (per-column statistics: type, non-null count, min, max, mean),
*schema* (print inferred CREATE TABLE DDL).
Mutually exclusive with a query argument, *--output*, *--explain*, and
Expand Down
40 changes: 26 additions & 14 deletions src/args.zig
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ pub fn printUsage(writer: *std.Io.Writer) !void {
\\ columns — list column names (use -v for types)
\\ validate — parse and print summary (exit 2 on error)
\\ sample [n] — print schema to stderr and n rows to stdout (default: 10)
\\ n must be a positive integer placed directly after the mode
\\ stats — per-column statistics (type, non-null, min, max, mean)
\\ schema — print inferred CREATE TABLE DDL
\\ --sample [<n>] (deprecated) Print schema and first <n> rows, use --inspect sample
Expand Down Expand Up @@ -331,7 +332,7 @@ pub fn printUsage(writer: *std.Io.Writer) !void {
\\ cat data.json | sql-pipe --input-format json 'SELECT * FROM t'
\\ cat data.ndjson | sql-pipe -I ndjson -O ndjson 'SELECT name FROM t WHERE age > 18'
\\ cat data.xml | sql-pipe -I xml --xml-root channel --xml-row item "SELECT title FROM t"
\\ cat data.csv | sql-pipe --sample 5
\\ cat data.csv | sql-pipe --inspect sample 5
\\ sql-pipe -f query.sql data.csv
\\ cat data.csv | sql-pipe -f query.sql
\\
Expand Down Expand Up @@ -428,6 +429,9 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP
var max_body_size: usize = 100 * 1024 * 1024;
var max_body_size_set = false;

// When set, the next iteration may consume an optional immediate sample count
var consume_sample_count = false;

var i: usize = 1;
while (i < args.len) : (i += 1) {
const arg = args[i];
Expand Down Expand Up @@ -488,9 +492,11 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP
if (i >= args.len) return error.MissingInspectMode;
const mode = InspectMode.parse(args[i]) catch return error.InvalidInspectMode;
inspect_mode = mode;
if (mode == .sample) consume_sample_count = true;
} else if (std.mem.startsWith(u8, arg, "--inspect=")) {
const mode = InspectMode.parse(arg["--inspect=".len..]) catch return error.InvalidInspectMode;
inspect_mode = mode;
if (mode == .sample) consume_sample_count = true;
} else if (std.mem.eql(u8, arg, "--columns")) {
inspect_mode = .columns;
inspect_deprecated = true;
Expand All @@ -500,19 +506,7 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP
} else if (std.mem.eql(u8, arg, "--sample")) {
inspect_mode = .sample;
inspect_deprecated = true;
// Peek at next arg: if it is a positive integer, consume it as the sample count
if (i + 1 < args.len) {
const next = args[i + 1];
if (next.len > 0 and next[0] != '-') {
if (std.fmt.parseUnsigned(usize, next, 10)) |n| {
if (n == 0) return error.InvalidSampleCount;
inspect_sample_n = n;
i += 1;
} else |_| {
// Not a number — keep default (10)
}
}
}
consume_sample_count = true;
} else if (std.mem.startsWith(u8, arg, "--sample=")) {
const val = arg["--sample=".len..];
const n = std.fmt.parseUnsigned(usize, val, 10) catch return error.InvalidSampleCount;
Expand Down Expand Up @@ -788,6 +782,24 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP
} else {
try positional_args.append(allocator, arg);
}

// Consume an optional sample count immediately after the mode, with the
// same rules as --sample: no flags, --, or non-numbers; zero is a usage error.
if (consume_sample_count) {
consume_sample_count = false;
if (i + 1 < args.len) {
const next = args[i + 1];
if (next.len > 0 and next[0] != '-') {
if (std.fmt.parseUnsigned(usize, next, 10)) |n| {
if (n == 0) return error.InvalidSampleCount;
inspect_sample_n = n;
i += 1;
} else |_| {
// Not a number — keep default (10)
}
}
}
}
}

// ─── Convert positional args to files + query ──────────────────────────
Expand Down
2 changes: 1 addition & 1 deletion src/main.zig
Original file line number Diff line number Diff line change
Expand Up @@ -694,7 +694,7 @@ pub fn main(init: std.process.Init.Minimal) void {
error.InvalidOutputFormat => fatal("unknown output format; supported: csv, tsv, json, ndjson, xml, markdown (md), html, sql", stderr_writer, .usage, .{}),
// ponytail: ColumnsWithQuery, ValidateWithQuery handled by InspectWithQuery above
error.InvalidOutputPath => fatal("--output requires a non-empty file path", stderr_writer, .usage, .{}),
error.InvalidSampleCount => fatal("--sample requires a positive integer value", stderr_writer, .usage, .{}),
error.InvalidSampleCount => fatal("sample count must be a positive integer (--inspect sample or --sample)", stderr_writer, .usage, .{}),
error.MissingQuery => {
stderr_writer.writeAll("error: no SQL query provided\n") catch |werr| std.log.err("failed to write error: {}", .{werr});
// Fall through to printUsage + exit below
Expand Down
Loading