Release v0.3.0: traffic and own files of each site

Makefile

@@ -37,7 +37,7 @@ TAR_OWNER := $(shell tar --version 2>/dev/null | grep -q GNU && echo '--owner=0
 
 # "go run pkg@version" builds each tool with the Go version of this module,
 # so a tool cannot be older than go.mod.
-GOLANGCI_LINT := go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.2
+GOLANGCI_LINT := go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.15.0
 GOVULNCHECK := go run golang.org/x/vuln/cmd/govulncheck@v1.8.0
 
 build: ## Build bin/fly for this platform

docs/monitoring-agent.md

@@ -4,7 +4,7 @@
 values to FlyWP. FlyWP shows them as charts and alerts, and can restart or update the agent without
 SSH.
 
-The agent follows the FlyWP monitoring agent contract **v0.5.0**. The contract defines the requests,
+The agent follows the FlyWP monitoring agent contract **v0.6.0**. The contract defines the requests,
 the fields and the rules on both sides.
 
 ## How it runs
@@ -40,6 +40,7 @@ After a change, run `sudo systemctl restart fly-agent`.
 | `ran.json` | The commands that ran, so that no command runs two times. |
 | `counters.json` | The last counters, so that a restart does not lose a minute of traffic. |
 | `state.json` | The report interval, and the time of the last release check. |
+| `traffic.json` | Where the next read of the proxy's traffic log starts: the file's inode, the byte offset, and a hash of the file's first bytes. |
 | `lock` | Stops a second agent. |
 
 Each file is written to a temporary file first, then renamed, so a crash never leaves half a file.
@@ -59,6 +60,7 @@ average of the minute hides.
 | Pressure (PSI) | the share of time that tasks waited for CPU, memory or disk I/O, and its peak. Not sent when the kernel has no PSI. |
 | Disk activity | bytes and operations read and written on the physical disks, and the peaks each second |
 | Sites | the CPU, memory and disk use of each site (see below) |
+| Traffic | the bytes the front proxy sent for each site name in the minute (see below) |
 
 With the samples, the agent sends the **status** of the server: restart needed, waiting updates
 (and security updates), OS, kernel, uptime, architecture, CPU count, Docker state (`running`,
@@ -76,6 +78,22 @@ For each site, the agent sends:
   the lowest I/O priority. A folder that the server user cannot list is skipped, so the value can be
   lower than the real use (for example, the database files in `~/.fly` belong to the container
   user).
+- **Own files:** the size of its `app` folder, from the same walk: the site's files without its
+  cache, backups and logs. `null` for a project with no `app` folder.
+
+### Traffic
+
+On a server where FlyWP set it up (White Label customer sites), the front proxy writes a second log
+for the agent: `~/.fly/nginx/vhost/flywp-traffic.log`, one line for each request with the name of the site
+(`$server_name`) and the bytes sent. Each sample adds the lines written since the sample before, by
+name, at most 1000 names.
+
+- The first read starts at the end of the file. The offset is saved before the sample is buffered,
+  so a stop loses those bytes and never sends them two times.
+- A new file (another inode, or other first bytes), or a file shorter than the offset, is read from
+  the start.
+- When the agent has read all of a file larger than 64 MiB, it cuts the file to 0.
+- Without the file, `traffic` is `null`.
 
 The agent reads Docker through its socket with two requests only: `GET /version` and
 `GET /containers/json`. It reads CPU and memory from cgroup v2. Without Docker or cgroup v2, it

internal/agent/wire/wire.go

@@ -1,5 +1,5 @@
 // Package wire holds the JSON bodies of the FlyWP monitoring agent contract
-// v0.5.0: the requests that the agent sends and the replies that it reads.
+// v0.6.0: the requests that the agent sends and the replies that it reads.
 package wire
 
 import (
@@ -94,15 +94,31 @@ type Sample struct {
 	// the agent cannot read Docker; an empty, non-nil slice ([]) means that
 	// Docker runs and no project matches.
 	Sites []Site `json:"sites"`
+
+	// Traffic holds the bytes that the server's front proxy sent for each
+	// site name since the sample before, from its traffic log
+	// ~/.fly/nginx/vhost/flywp-traffic.log (contract v0.6.0). nil (JSON null) means that the server has no traffic log; an
+	// empty, non-nil slice means that no request came.
+	Traffic []Traffic `json:"traffic"`
 }
 
 // Site is the use of one Docker Compose project in the minute. nil (JSON
-// null) means "not known". DiskUsedBytes is set in one sample each hour.
+// null) means "not known". DiskUsedBytes and FilesBytes are set in one
+// sample each hour, from the same walk: FilesBytes is the project's app
+// folder, the site's own files (contract v0.6.0).
 type Site struct {
 	Directory       string   `json:"directory"`
 	CPUPercent      *float64 `json:"cpu_percent"`
 	MemoryUsedBytes *uint64  `json:"memory_used_bytes"`
 	DiskUsedBytes   *uint64  `json:"disk_used_bytes"`
+	FilesBytes      *uint64  `json:"files_bytes"`
+}
+
+// Traffic is what the front proxy sent for one site name: the name of its
+// matched server block (nginx $server_name), and the bytes with the headers.
+type Traffic struct {
+	Host  string `json:"host"`
+	Bytes uint64 `json:"bytes"`
 }
 
 // MetricsReply is the reply to POST /agent/v1/metrics.

internal/metrics/diskusage_other.go

@@ -5,11 +5,17 @@ package metrics
 import (
 	"context"
 	"errors"
+	"io/fs"
+	"os"
 	"runtime"
 )
 
 func fileID(string) uint64 { return 0 }
 
-func diskUsage(context.Context, string) (uint64, int, error) {
-	return 0, 0, errors.New("disk use is not measured on " + runtime.GOOS)
+func openNoFollow(path string) (*os.File, error) { return os.OpenFile(path, os.O_RDWR, 0) }
+
+func inodeOf(fs.FileInfo) uint64 { return 0 }
+
+func diskUsage(context.Context, string) (folderUse, int, error) {
+	return folderUse{}, 0, errors.New("disk use is not measured on " + runtime.GOOS)
 }

internal/metrics/diskusage_unix.go

@@ -5,10 +5,25 @@ package metrics
 import (
 	"context"
 	"io/fs"
+	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 )
 
+// openNoFollow opens a file to read and cut it, and refuses a symbolic link.
+func openNoFollow(path string) (*os.File, error) {
+	return os.OpenFile(path, os.O_RDWR|syscall.O_NOFOLLOW, 0)
+}
+
+// inodeOf returns the inode of an open file's info, or 0.
+func inodeOf(info fs.FileInfo) uint64 {
+	if st, ok := info.Sys().(*syscall.Stat_t); ok {
+		return uint64(st.Ino)
+	}
+	return 0
+}
+
 // fileID returns the inode of a path, or 0.
 func fileID(path string) uint64 {
 	var st syscall.Stat_t
@@ -19,13 +34,15 @@ func fileID(path string) uint64 {
 }
 
 // diskUsage returns the space that the files under root take on the disk: the
-// allocated blocks, as du shows them (st_blocks × 512). It does not follow a
-// symbolic link, does not go into an other file system, and counts a file
-// with more than one hard link one time. It skips what it cannot read, and
-// returns the number of those skips. It stops when ctx is done.
-func diskUsage(ctx context.Context, root string) (used uint64, skipped int, err error) {
-	var dev uint64
+// allocated blocks, as du shows them (st_blocks × 512), and the part of it in
+// the app folder of root, the site's own files (nil without one). It does not
+// follow a symbolic link, does not go into an other file system, and counts a
+// file with more than one hard link one time. It skips what it cannot read,
+// and returns the number of those skips. It stops when ctx is done.
+func diskUsage(ctx context.Context, root string) (use folderUse, skipped int, err error) {
+	var dev, own uint64
 	seen := map[[2]uint64]bool{}
+	app := filepath.Join(root, "app")
 
 	err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
 		if ctxErr := ctx.Err(); ctxErr != nil {
@@ -69,9 +86,16 @@ func diskUsage(ctx context.Context, root string) (used uint64, skipped int, err
 			seen[key] = true
 		}
 
-		used += uint64(st.Blocks) * 512
+		blocks := uint64(st.Blocks) * 512
+		use.used += blocks
+		if path == app && d.IsDir() {
+			use.own = &own
+		}
+		if path == app || strings.HasPrefix(path, app+string(filepath.Separator)) {
+			own += blocks
+		}
 		return nil
 	})
 
-	return used, skipped, err
+	return use, skipped, err
 }

internal/metrics/diskusage_unix_test.go

@@ -50,12 +50,33 @@ func TestDiskUsage(t *testing.T) {
 		want += blocks(t, filepath.Join(root, p))
 	}
 
+	var wantOwn uint64
+	for _, p := range []string{"app", "app/uploads", "app/index.php", "app/uploads/big.jpg"} {
+		wantOwn += blocks(t, filepath.Join(root, p))
+	}
+
 	got, skipped, err := diskUsage(context.Background(), root)
 	if err != nil || skipped != 0 {
-		t.Fatalf("diskUsage() = %d, %d skipped, %v", got, skipped, err)
+		t.Fatalf("diskUsage() = %d, %d skipped, %v", got.used, skipped, err)
+	}
+	if got.used != want {
+		t.Errorf("diskUsage() = %d, want %d: the blocks of each file one time, without the target of the symbolic link", got.used, want)
+	}
+	// The site's own files: the app folder (contract v0.6.0).
+	if got.own == nil || *got.own != wantOwn {
+		t.Errorf("diskUsage() own = %v, want %d: the app folder", got.own, wantOwn)
+	}
+}
+
+func TestDiskUsageWithNoAppFolder(t *testing.T) {
+	root := t.TempDir()
+	if err := os.WriteFile(filepath.Join(root, "docker-compose.yml"), []byte("services: {}"), 0o644); err != nil {
+		t.Fatal(err)
 	}
-	if got != want {
-		t.Errorf("diskUsage() = %d, want %d: the blocks of each file one time, without the target of the symbolic link", got, want)
+
+	got, _, err := diskUsage(context.Background(), root)
+	if err != nil || got.used == 0 || got.own != nil {
+		t.Errorf("diskUsage() = %+v, %v, want a size and no own files", got, err)
 	}
 }
 
@@ -80,8 +101,8 @@ func TestDiskUsageSkipsWhatItCannotRead(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if skipped != 1 || got != blocks(t, root)+blocks(t, locked) {
-		t.Errorf("diskUsage() = %d with %d skipped, want the two folders and 1 skip", got, skipped)
+	if skipped != 1 || got.used != blocks(t, root)+blocks(t, locked) {
+		t.Errorf("diskUsage() = %d with %d skipped, want the two folders and 1 skip", got.used, skipped)
 	}
 }
 

internal/metrics/metrics.go

@@ -1,7 +1,7 @@
 // Package metrics measures a Linux server for the monitoring agent: CPU,
 // load, memory, swap, disk, network, pressure (PSI) and disk activity each
 // minute, with the peaks of the minute from a reading each 10 seconds, the
-// use of each site, and the status of the server (contract v0.5.0). It needs
+// use of each site, and the status of the server (contract v0.5.0; own files and traffic v0.6.0). It needs
 // no root.
 package metrics
 
@@ -99,6 +99,8 @@ type Collector struct {
 	// and walker measures the disk use of the sites.
 	prevContainers *containerReadings
 	walker         *diskWalker
+	// traffic reads the front proxy's traffic log (contract v0.6.0).
+	traffic *trafficReader
 
 	updatesAt       time.Time
 	updatesKnown    bool
@@ -121,6 +123,10 @@ func New(root, stateDir string, log *slog.Logger) *Collector {
 	c.docker = dockerapi.New(c.file("var/run/docker.sock"))
 	c.home = homeDir()
 	c.walker = newDiskWalker()
+	c.traffic = &trafficReader{state: filepath.Join(stateDir, "traffic.json"), limit: trafficLimit}
+	if c.home != "" {
+		c.traffic.path = filepath.Join(c.home, ".fly", "nginx", "vhost", "flywp-traffic.log")
+	}
 
 	// Take a reading now, so that the first sample has a CPU value for the
 	// time since the start. The saved reading comes before it only when it is
@@ -242,6 +248,9 @@ func (c *Collector) Sample(now time.Time) (wire.Sample, error) {
 	// The sites come after each step that can fail: they take the result
 	// of the disk walk, which must not go with a sample that is dropped.
 	s.Sites = c.sites(context.Background(), cur, readAt)
+	// Last too: the read saves its offset, so its bytes go with this sample
+	// or with none.
+	s.Traffic = c.traffic.read(c.log)
 
 	c.prev = &cur
 	c.readings = nil

internal/metrics/sites.go

@@ -58,7 +58,7 @@ func homeDir() string {
 }
 
 // sites measures each Docker Compose project in the home folder at the tick
-// cur, read at readAt (contract v0.5.0). It returns nil when the agent cannot read Docker or
+// cur, read at readAt (contract v0.5.0; own files and traffic v0.6.0). It returns nil when the agent cannot read Docker or
 // the cgroups: Docker does not run, the socket refuses the agent, or the
 // server has cgroup v1. It returns an empty, non-nil slice when no project
 // matches.
@@ -148,7 +148,8 @@ func (c *Collector) sites(ctx context.Context, cur reading, readAt time.Time) []
 			site.MemoryUsedBytes = &p.mem
 		}
 		if d, ok := disk[name]; ok {
-			site.DiskUsedBytes = &d
+			site.DiskUsedBytes = &d.used
+			site.FilesBytes = d.own
 		}
 		out = append(out, site)
 	}
@@ -228,21 +229,28 @@ func statValue(key string) func([]byte) (uint64, error) {
 	}
 }
 
+// folderUse is the disk use of one project folder: all of it, and its app
+// folder (the site's own files), nil when it has none (contract v0.6.0).
+type folderUse struct {
+	used uint64
+	own  *uint64
+}
+
 // diskWalker measures the disk use of the project folders in a goroutine of
 // its own, so that a large folder does not delay the samples. The collector
 // takes each result one time.
 type diskWalker struct {
 	mu      sync.Mutex
 	running bool
-	last    time.Time         // the start of the last walk
-	results map[string]uint64 // by directory, not yet sent
+	last    time.Time            // the start of the last walk
+	results map[string]folderUse // by directory, not yet sent
 	// skipWarned holds the directories whose skipped files were logged as a
 	// warning. Only the walk goroutine uses it.
 	skipWarned map[string]bool
 	// done is closed when the running walk ends. Tests wait for it.
 	done chan struct{}
 	// walk measures one folder, and timeout limits it. Tests replace them.
-	walk    func(ctx context.Context, root string) (uint64, int, error)
+	walk    func(ctx context.Context, root string) (folderUse, int, error)
 	timeout time.Duration
 }
 
@@ -265,7 +273,7 @@ func (w *diskWalker) start(folders map[string]string, log *slog.Logger) {
 	go func() {
 		defer close(done)
 
-		results := map[string]uint64{}
+		results := map[string]folderUse{}
 		for _, name := range slices.Sorted(func(yield func(string) bool) {
 			for name := range folders {
 				if !yield(name) {
@@ -302,12 +310,12 @@ func (w *diskWalker) start(folders map[string]string, log *slog.Logger) {
 // for it after the timeout. A walk that hangs in a system call, for example on
 // a network mount that does not answer, stays behind; the other folders and
 // the next walks go on.
-func (w *diskWalker) walkOne(root string) (uint64, int, error) {
+func (w *diskWalker) walkOne(root string) (folderUse, int, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), w.timeout)
 	defer cancel()
 
 	type result struct {
-		used    uint64
+		used    folderUse
 		skipped int
 		err     error
 	}
@@ -325,12 +333,12 @@ func (w *diskWalker) walkOne(root string) (uint64, int, error) {
 	case r := <-ch:
 		return r.used, r.skipped, r.err
 	case <-ctx.Done():
-		return 0, 0, ctx.Err()
+		return folderUse{}, 0, ctx.Err()
 	}
 }
 
 // take returns the results of the last walk that ended, one time.
-func (w *diskWalker) take() map[string]uint64 {
+func (w *diskWalker) take() map[string]folderUse {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	r := w.results