Repository navigation
Expand file tree
/
Copy pathMakefile
More file actions
141 lines (119 loc) · 5.13 KB
/
Copy pathMakefile
File metadata and controls
141 lines (119 loc) · 5.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
.PHONY: all data extract up up-pgcache bench-up bench-down bench-run bench-warmup down load psql psql-cached clean reset nuke-slot help
DATA_DIR := data
DUMP_URL := https://archive.org/download/stackexchange/dba.stackexchange.com.7z
DUMP := $(DATA_DIR)/dba.stackexchange.com.7z
SENTINEL := $(DATA_DIR)/Posts.xml
PGHOST ?= localhost
PGPORT ?= 55432
PGUSER ?= pgcache
PGPASSWORD ?= pgcache
PGDATABASE ?= dba_se
export PGHOST PGPORT PGUSER PGPASSWORD PGDATABASE
help:
@echo "Setup:"
@echo " data download and extract the dba.stackexchange.com dump"
@echo " up start the origin Postgres container"
@echo " up-pgcache start origin + pgcache"
@echo " load load the dump into the running origin (requires up + data)"
@echo " all data + up + load"
@echo
@echo "Benchmark:"
@echo " bench-up full stack: origin + pgcache + Prometheus + postgres-exporter"
@echo " bench-warmup 30s unmeasured warmup of the pgcache cache"
@echo " bench-run drive both lanes (override with DURATION=, CONCURRENCY=, SEED=, WRITE_RATE=, TAG_GROW_FRACTION=)"
@echo " bench-down stop the full stack"
@echo
@echo "Utilities:"
@echo " psql open psql against the origin"
@echo " psql-cached open psql against pgcache (proxied to origin)"
@echo " reset drop and recreate the origin database (container stays up)"
@echo " nuke-slot drop pgcache replication slot and publication on the origin"
@echo " down stop all containers"
@echo " clean stop containers, drop volumes, remove extracted XML and loader"
all: data up load
@echo "==> origin loaded; connect with: make psql"
data: $(SENTINEL)
$(DUMP):
mkdir -p $(DATA_DIR)
curl -L --progress-bar -o $@ $(DUMP_URL)
$(SENTINEL): $(DUMP)
cd $(DATA_DIR) && 7z x -y dba.stackexchange.com.7z
touch $(SENTINEL)
up:
docker compose up -d origin
./load/wait_ready.sh
up-pgcache: up
docker compose up -d pgcache
./load/wait_pgcache.sh
# Bring up the full stack: origin, pgcache, Prometheus, postgres-exporter.
# The driver itself runs on the host (see `make bench-run`) and is scraped via
# host.docker.internal.
bench-up:
docker compose up -d origin
./load/wait_ready.sh
docker compose up -d pgcache
./load/wait_pgcache.sh
docker compose --profile observability up -d
@echo
@echo "==> Prometheus: http://localhost:59091"
# Stop the full stack. Drops the slot first so iterative dev doesn't
# leave WAL accumulating on origin.
bench-down: nuke-slot
docker compose --profile observability --profile tools down
# Build (if needed) and run the driver against both lanes for DURATION.
# Override: `make bench-run DURATION=60s CONCURRENCY=200 SEED=7 WRITE_RATE=0.05`
DURATION ?= 5m
CONCURRENCY ?= 50
SEED ?= 42
WRITE_RATE ?= 0.02
TAG_GROW_FRACTION ?= 0.2
bench-run:
cd driver && cargo build --release --quiet
driver/target/release/pgcache-bench-driver \
--target lane=direct,dsn=postgres://pgcache:pgcache@localhost:55432/dba_se \
--target lane=cached,dsn=postgres://pgcache:pgcache@localhost:55433/dba_se \
--concurrency $(CONCURRENCY) \
--duration $(DURATION) \
--seed $(SEED) \
--write-rate $(WRITE_RATE) \
--tag-grow-fraction $(TAG_GROW_FRACTION) \
--metrics-addr 0.0.0.0:9100
# Drive only the cached lane at low concurrency to populate pgcache before
# the measured run. Without this, the first seconds of `bench-run` measure
# cold-cache population (every page is a miss), not steady-state serving.
WARMUP_DURATION ?= 30s
WARMUP_CONCURRENCY ?= 20
bench-warmup:
cd driver && cargo build --release --quiet
driver/target/release/pgcache-bench-driver \
--target lane=cached,dsn=postgres://pgcache:pgcache@localhost:55433/dba_se \
--concurrency $(WARMUP_CONCURRENCY) \
--duration $(WARMUP_DURATION) \
--seed $(SEED) \
--write-rate 0 \
--metrics-addr 0.0.0.0:9100
down:
docker compose --profile observability --profile tools down
load: $(SENTINEL)
./load/load.sh
psql:
PGPASSWORD=$(PGPASSWORD) psql -h $(PGHOST) -p $(PGPORT) -U $(PGUSER) -d $(PGDATABASE)
# Open psql through pgcache. Same data as `make psql` but every read flows
# through the cache lane.
psql-cached:
PGPASSWORD=$(PGPASSWORD) psql -h $(PGHOST) -p 55433 -U $(PGUSER) -d $(PGDATABASE)
reset:
PGPASSWORD=$(PGPASSWORD) psql -h $(PGHOST) -p $(PGPORT) -U $(PGUSER) -d postgres \
-c "DROP DATABASE IF EXISTS $(PGDATABASE);" \
-c "CREATE DATABASE $(PGDATABASE);"
# Drop the pgcache replication slot and publication on the origin. Run before
# `make clean` (or any time you docker-compose-down the pgcache container) so
# the origin doesn't accumulate WAL forever waiting for a consumer that's
# never coming back.
nuke-slot:
-PGPASSWORD=$(PGPASSWORD) psql -h $(PGHOST) -p $(PGPORT) -U $(PGUSER) -d $(PGDATABASE) \
-c "SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name LIKE 'pgcache_slot%';" \
-c "DROP PUBLICATION IF EXISTS pgcache_pub;"
clean: nuke-slot
docker compose down -v
rm -rf $(DATA_DIR)/*.xml load/stackexchange-dump-to-postgres load/.venv load/.stage