From 0e571c572a8232c636994005e5ce7595e904e781 Mon Sep 17 00:00:00 2001 From: selul Date: Wed, 4 Nov 2026 15:00:17 +0200 Subject: [PATCH 1/9] Add "optimole export": export the library and download its archives New command group: - export start: start an export of the whole library (or follow the one already running), wait for it and download each ZIP archive as soon as the server lists it; --no-wait only starts it - export download [ID|latest]: download or resume an export - export status, export list, export delete Archives land in DIR/optimole-export-ID/DOMAIN/images-N.zip next to an export.json manifest (no URLs). Each archive goes to a .part file named after its listed size, is resumed with HTTP Range after a cut or a stall, gets a fresh URL when storage refuses one, and is renamed into place only once its size matches and it opens as a ZIP. An archive the server rewrote (new size) is downloaded again. Domain names from the server are turned into safe, distinct folder names on every OS. Conflicts on start use the dashboard's new export_in_progress and export_limit_reached codes and fall back to the export list on servers that answer a bare 409. Download URLs never get the API key, are never printed and redirects are refused, as for uploads. The fake server gains the export endpoints and an archive store with Range support and fault injection (cuts, stalls, ignored ranges). Co-Authored-By: Claude Opus 5.5 --- README.md | 110 +++++- cmd/optimole/main.go | 3 +- internal/api/client.go | 10 +- internal/api/errors.go | 28 +- internal/api/exports.go | 218 ++++++++++ internal/api/exports_test.go | 225 +++++++++++ internal/cli/cli.go | 5 +- internal/cli/export.go | 635 ++++++++++++++++++++++++++++++ internal/cli/export_test.go | 358 +++++++++++++++++ internal/export/download.go | 315 +++++++++++++++ internal/export/export.go | 467 ++++++++++++++++++++++ internal/export/export_test.go | 624 +++++++++++++++++++++++++++++ internal/export/manifest.go | 84 ++++ internal/export/names.go | 79 ++++ internal/output/export.go | 209 ++++++++++ internal/output/output.go | 5 +- internal/testserver/exports.go | 474 ++++++++++++++++++++++ internal/testserver/testserver.go | 33 +- 18 files changed, 3862 insertions(+), 20 deletions(-) create mode 100644 internal/api/exports.go create mode 100644 internal/api/exports_test.go create mode 100644 internal/cli/export.go create mode 100644 internal/cli/export_test.go create mode 100644 internal/export/download.go create mode 100644 internal/export/export.go create mode 100644 internal/export/export_test.go create mode 100644 internal/export/manifest.go create mode 100644 internal/export/names.go create mode 100644 internal/output/export.go create mode 100644 internal/testserver/exports.go diff --git a/README.md b/README.md index 0293697..af947b1 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ # optimole `optimole` uploads local files into an [Optimole](https://optimole.com) media library, into the folders you -choose. It is a single binary with no dependencies. +choose, and exports the library back as ZIP archives. It is a single binary with no dependencies. ```sh optimole login # paste your API key once optimole upload -r ./photos --folder Campaigns # photos/2024/a.jpg -> Campaigns/2024 +optimole export start # export the library, download ./optimole-export-ID ``` ## Install @@ -67,7 +68,7 @@ the storage URL the file bytes go to. **Folder-scoped keys** (the ones starting with `optml-s`) work for uploads into the folders they were granted (and their subfolders): always pass `--folder` (or `--folder-id`). They cannot create folders -(`folder_creation_forbidden`), upload into the library root (`folder_required`), or list folders. +(`folder_creation_forbidden`), upload into the library root (`folder_required`), list folders, or export. The API base URL defaults to `https://dashboard.optimole.com/api`; override it with `--api-url` or `OPTIMOLE_API_URL` (for example to try another environment). Only `https` URLs are @@ -242,6 +243,103 @@ ID PATH Folder-scoped keys cannot list folders (the API answers 403); `upload --folder PATH` still works with them. +### `optimole export ` + +Exports the original files of the whole library and downloads them. An export covers every whitelisted +domain of the account, plus `directUpload` for the files uploaded straight to the library; the server zips +each domain into archives of about 1 GB. Exports need a full-access API key. + +| Command | What it does | +|---|---| +| `optimole export start [flags]` | Starts an export, follows it and downloads each archive as soon as it is ready. If an export is already running, follows that one instead. `--no-wait` only starts it and prints its ID. | +| `optimole export download [ID\|latest] [flags]` | Downloads an existing export (default: the latest), waiting while it runs. Run it again to resume. | +| `optimole export status [ID\|latest] [--json]` | Shows the export's status and each domain's status, archives and size. | +| `optimole export list [--json]` | Lists the account's exports, newest first. | +| `optimole export delete ID` | Deletes a finished export and its archives on the server (local files are not touched). | + +`start` and `download` take: + +| Flag | Meaning | +|---|---| +| `-o`, `--output DIR` | Where the export folder goes (default: the current folder). | +| `--domain NAME` | Only download this domain's archives (repeatable, case-insensitive). The export itself always covers every domain. | +| `-c`, `--concurrency N` | Archives downloaded in parallel, 1-8 (default 2). | +| `--timeout DURATION` | Give up after this long, e.g. `90m` or `2h` (default: no limit). | +| `--json` | One JSON object per event and a final summary object. | + +```sh +optimole export start # start, wait, download everything +optimole export start --no-wait # start it now, download later +optimole export download # the latest export, into ./optimole-export-ID +optimole export download 42 -o /backups --domain example.com +``` + +#### Where the files go + +```text +optimole-export-42/ + example.com/images-1.zip + example.com/images-2.zip + directUpload/images-1.zip + export.json +``` + +Each domain gets a folder named after it. Domain names come from the server, so the CLI keeps only letters, +digits, `.`, `-` and `_` (anything else becomes `_`, so `*.example.com` becomes `_.example.com`), drops leading +and trailing dots, avoids Windows device names, and adds `-2`, `-3`... when two names differ only in case. +`export.json` maps each domain to its folder and lists its archives with their size and whether they are +downloaded. It holds no download URLs. The archives are not unpacked. + +#### How downloads work + +- While the export runs, the CLI checks it every 5 seconds (less often after a few minutes) and downloads each + archive as soon as the server lists it. +- An archive is written to `images-N.zip..part` and renamed into place only when its size matches what + the server listed and it opens as a ZIP. A download that gets cut is resumed where it stopped; one that + stalls for 2 minutes is restarted. An archive that is not a valid ZIP is downloaded once more, then reported + as failed. +- Running `download` again skips the archives already in place and resumes the partial ones. If the server + listed an archive with a new size (it rewrote it), that archive is downloaded again. +- Download URLs are valid for a day; when storage refuses one, the CLI asks the server for a fresh one. + Download URLs never get the API key and are never printed, and redirects are refused. +- Ctrl-C and `--timeout` stop only the CLI. The export continues on the server, and the command prints how to + resume (`optimole export download ID`). There is no way to cancel an export from the API. + +#### Output + +```text +Started export 42. +Export 42: running + example.com: running + directUpload: running +downloading optimole-export-42/example.com/images-1.zip (1.0 GB) +downloaded optimole-export-42/example.com/images-1.zip (1.0 GB in 41s) + directUpload: completed, no files +... +Export 42: completed + +Export 42 completed: 2 downloaded, 0 already there, 0 failed (1.8 GB written) in optimole-export-42. +``` + +With `--json`: + +```json +{"event":"started","id":42,"created":true} +{"event":"export","id":42,"status":"running"} +{"event":"domain","domain":"example.com","status":"running","archives":0} +{"event":"downloading","domain":"example.com","part":1,"path":"optimole-export-42/example.com/images-1.zip","size":1073741824} +{"event":"archive","domain":"example.com","part":1,"path":"optimole-export-42/example.com/images-1.zip","status":"downloaded","size":1073741824,"seconds":41} +{"summary":{"id":42,"status":"completed","dir":"optimole-export-42","downloaded":2,"present":0,"failed":0,"bytes":1932735283,"failed_domains":[],"interrupted":false,"timed_out":false}} +``` + +An archive's `status` is `downloaded`, `present` (already on disk) or `failed` (with `error`). `created` is +`false` when the command follows an export that was already running. + +An export is `queued`, `running`, `completed`, `failed` or `cancelled`. Once every domain has finished, it is +`failed` if one of them failed; the other domains' archives are still downloaded, and the command exits 1. An account keeps at most 5 +exports (`export_limit_reached`); delete one with `optimole export delete ID`, or wait: finished exports are +deleted automatically 10 days after they finish. + ### `optimole login`, `optimole logout` `login` reads the key from stdin: type or paste it at the prompt, or pipe it (`optimole login < key.txt`; @@ -258,9 +356,9 @@ Prints the version, commit and build date. | Code | Meaning | |---|---| -| 0 | Every selected file was uploaded (skipped files do not count). | -| 1 | At least one file failed, there was nothing to upload, or the output could not be written. | -| 2 | Usage or configuration error (bad flags, a directory without `-r`, missing key, non-https URL). Nothing was uploaded. | +| 0 | Every selected file was uploaded (skipped files do not count). For `export start` and `export download`: the export completed and every selected archive is on disk. | +| 1 | At least one file failed, there was nothing to upload, or the output could not be written. For exports: the export failed or was cancelled, an archive failed, `--timeout` passed, or the account keeps the most exports it may. | +| 2 | Usage or configuration error (bad flags, a directory without `-r`, an unknown `--domain`, missing key, non-https URL). Nothing was uploaded or downloaded. | | 130 | Interrupted with Ctrl-C. | ## Limits @@ -271,6 +369,8 @@ Prints the version, commit and build date. - The API allows 300 requests per minute per account; each file takes 2 API requests plus the storage upload, so very large batches may see `429` answers, which the CLI waits out. - A presigned upload URL is valid for 15 minutes. +- An account runs one export at a time and keeps at most 5. Export archives are about 1 GB each (a part can be + larger when its first files are), and their download URLs are valid for a day. ## Development diff --git a/cmd/optimole/main.go b/cmd/optimole/main.go index b3d4cfc..60c0c02 100644 --- a/cmd/optimole/main.go +++ b/cmd/optimole/main.go @@ -1,4 +1,5 @@ -// Command optimole uploads local files into an Optimole media library. +// Command optimole uploads local files into an Optimole media library and +// exports the library as ZIP archives. package main import ( diff --git a/internal/api/client.go b/internal/api/client.go index b428c3d..394556e 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -1,6 +1,7 @@ // Package api is a small client for the parts of the Optimole dashboard API // the CLI uses: reserving an asset, sending its bytes to the presigned storage -// URL, completing it, and listing folders. +// URL, completing it, listing folders, and running asset exports and +// downloading their archives. package api import ( @@ -108,9 +109,10 @@ func New(opts Options) (*Client, error) { // GET or send the key somewhere unexpected. CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }, - // No overall timeout: a 100 MB body can take a while. The context and - // the transport's header timeout bound it instead. Redirects are - // refused: following one would replay the file to another host. + // No overall timeout: a 100 MB upload or a 1 GB archive download can + // take a while. The context and the transport's header timeout bound + // it instead. Redirects are refused: following one would replay the + // file to another host. storage: &http.Client{ Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, diff --git a/internal/api/errors.go b/internal/api/errors.go index f8dcb8a..fc4b44e 100644 --- a/internal/api/errors.go +++ b/internal/api/errors.go @@ -23,6 +23,11 @@ const ( CodeFolderRequired = "folder_required" // CodeFolderCreationForbidden: a folder-scoped key asked to create folders. CodeFolderCreationForbidden = "folder_creation_forbidden" + // CodeExportInProgress: an export is already queued or running (its ID + // comes in ExportID). + CodeExportInProgress = "export_in_progress" + // CodeExportLimitReached: the account keeps the most exports it may. + CodeExportLimitReached = "export_limit_reached" ) // Error is a non-2xx answer from the dashboard API. @@ -34,8 +39,10 @@ type Error struct { Message string // Fields holds the validation messages of a 422, keyed by field. Fields map[string][]string - // Limit and Count come with upload_limit_reached. + // Limit and Count come with upload_limit_reached and export_limit_reached. Limit, Count FlexInt + // ExportID comes with export_in_progress. + ExportID FlexInt // Expected and Actual come with upload_size_mismatch. Expected, Actual FlexInt // RetryAfter is the parsed Retry-After header, if any. @@ -56,6 +63,9 @@ func (e *Error) Error() string { if e.Limit.Valid || e.Count.Valid { details = append(details, fmt.Sprintf("limit %s, used %s", flexString(e.Limit), flexString(e.Count))) } + if e.ExportID.Valid { + details = append(details, fmt.Sprintf("export %d", e.ExportID.Value)) + } if e.Expected.Valid || e.Actual.Valid { details = append(details, fmt.Sprintf("expected %s bytes, storage has %s", flexString(e.Expected), flexString(e.Actual))) } @@ -116,14 +126,16 @@ func (e *NetworkError) Error() string { func (e *NetworkError) Unwrap() error { return e.Err } -// StorageError is a failed PUT to the presigned storage URL. +// StorageError is a failed request to a presigned storage URL. type StorageError struct { + // Op names the request; empty means the upload PUT. + Op string // StatusCode is 0 when no HTTP answer arrived (network error). StatusCode int Code string Message string Err error - // Attempts is how many PUTs were made to the URL. + // Attempts is how many requests were made to the URL. Attempts int // Uncertain is set when an attempt got no answer or a 5xx, so the bytes // may have been stored under this asset after all. @@ -156,10 +168,14 @@ func (e *StorageError) Error() string { if e.Expired { extra = append(extra, "the upload URL expired") } + op := e.Op + if op == "" { + op = "upload to storage" + } if len(extra) == 0 { - return "upload to storage: " + msg + return op + ": " + msg } - return fmt.Sprintf("upload to storage: %s (%s)", msg, strings.Join(extra, "; ")) + return fmt.Sprintf("%s: %s (%s)", op, msg, strings.Join(extra, "; ")) } func (e *StorageError) Unwrap() error { return e.Err } @@ -193,6 +209,7 @@ func parseError(op string, resp *http.Response, body []byte, now time.Time) *Err Count FlexInt `json:"count"` Expected FlexInt `json:"expected"` Actual FlexInt `json:"actual"` + ExportID FlexInt `json:"export_id"` } if err := json.Unmarshal(body, &payload); err != nil { ct := resp.Header.Get("Content-Type") @@ -206,6 +223,7 @@ func parseError(op string, resp *http.Response, body []byte, now time.Time) *Err e.Code = payload.Code e.Limit, e.Count = payload.Limit, payload.Count e.Expected, e.Actual = payload.Expected, payload.Actual + e.ExportID = payload.ExportID if len(payload.Errors) > 0 { e.Fields = make(map[string][]string, len(payload.Errors)) for field, raw := range payload.Errors { diff --git a/internal/api/exports.go b/internal/api/exports.go new file mode 100644 index 0000000..cd4c1de --- /dev/null +++ b/internal/api/exports.go @@ -0,0 +1,218 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strconv" +) + +// Operation names of the export calls, used as the Op of errors. +const ( + OpCreateExport = "create export" + OpGetExport = "get export" + OpListExports = "list exports" + OpDeleteExport = "delete export" + OpDownloadArchive = "download archive" +) + +// Export statuses. The last three are final. +const ( + ExportQueued = "queued" + ExportRunning = "running" + ExportCompleted = "completed" + ExportFailed = "failed" + ExportCancelled = "cancelled" +) + +// IsFinalStatus reports whether an export or domain status will not change +// any more. +func IsFinalStatus(status string) bool { + switch status { + case ExportCompleted, ExportFailed, ExportCancelled: + return true + } + return false +} + +// Export is an asset export. Listing and creating answer with the domain +// names only; GET /asset-exports/{id} adds each domain's status and archives. +type Export struct { + ID int64 `json:"id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + Domains []ExportDomain `json:"domains"` +} + +// ExportDomain is one domain of an export ("directUpload" holds the files +// uploaded straight to the library). +type ExportDomain struct { + Domain string `json:"domain"` + // Status is empty when the answer carried only the name. + Status string `json:"status"` + Parts []ExportPart `json:"zip_files"` +} + +// UnmarshalJSON accepts a bare domain name as well as a domain object. +func (d *ExportDomain) UnmarshalJSON(b []byte) error { + var name string + if err := json.Unmarshal(b, &name); err == nil { + *d = ExportDomain{Domain: name} + return nil + } + type plain ExportDomain + var p plain + if err := json.Unmarshal(b, &p); err != nil { + return err + } + *d = ExportDomain(p) + return nil +} + +// ExportPart is one finished ZIP archive of a domain. +type ExportPart struct { + Number FlexInt `json:"part_number"` + // URL is a presigned storage URL, valid for a day. It is a credential + // for that time: never print it. + URL string `json:"url"` + Size FlexInt `json:"size"` +} + +// ExportQuery holds the query parameters of GET /asset-exports. +type ExportQuery struct { + PerPage int + Cursor string +} + +// ExportPage is one cursor page of GET /asset-exports, newest first. +type ExportPage struct { + Data []Export `json:"data"` + Meta struct { + NextCursor *string `json:"next_cursor"` + HasMore bool `json:"has_more"` + } `json:"meta"` +} + +// CreateExport starts an export of the whole library. A conflict (one +// already running, or the account keeps the most exports it may) is an +// *Error with status 409 and, on current servers, CodeExportInProgress or +// CodeExportLimitReached. +// +// It is retried like asset creation, only when nothing can have been +// created: a second export would take one of the account's few slots. +func (c *Client) CreateExport(ctx context.Context) (*Export, error) { + var out Export + if err := c.doJSON(ctx, OpCreateExport, http.MethodPost, c.endpoint("asset-exports"), struct{}{}, &out, retryUncommitted); err != nil { + return nil, err + } + if out.ID <= 0 { + return nil, fmt.Errorf("%s: the server returned no export ID", OpCreateExport) + } + return &out, nil +} + +// GetExport returns an export with its domains and finished archives. +func (c *Client) GetExport(ctx context.Context, id int64) (*Export, error) { + var out Export + if err := c.doJSON(ctx, OpGetExport, http.MethodGet, c.exportEndpoint(id), nil, &out, retryIdempotent); err != nil { + return nil, err + } + return &out, nil +} + +// ListExports returns one page of the account's exports. +func (c *Client) ListExports(ctx context.Context, q ExportQuery) (*ExportPage, error) { + u := c.endpoint("asset-exports") + v := url.Values{} + if q.PerPage > 0 { + v.Set("per_page", strconv.Itoa(q.PerPage)) + } + if q.Cursor != "" { + v.Set("cursor", q.Cursor) + } + u.RawQuery = v.Encode() + var page ExportPage + if err := c.doJSON(ctx, OpListExports, http.MethodGet, u, nil, &page, retryIdempotent); err != nil { + return nil, err + } + return &page, nil +} + +// DeleteExport deletes a finished export and its archives. A 404 on a +// repeated call counts as success: an earlier attempt that got no answer +// deleted it. +func (c *Client) DeleteExport(ctx context.Context, id int64) error { + err := c.doJSON(ctx, OpDeleteExport, http.MethodDelete, c.exportEndpoint(id), nil, nil, retryIdempotent) + var apiErr *Error + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound && apiErr.Attempts > 1 { + return nil + } + return err +} + +func (c *Client) exportEndpoint(id int64) *url.URL { + return c.endpoint("asset-exports", strconv.FormatInt(id, 10)) +} + +// Object is the answer to a presigned storage GET. +type Object struct { + Body io.ReadCloser + // Offset is the byte the body starts at: the requested offset when + // storage honoured the range, 0 when it sent the whole object. + Offset int64 + // Size is the object's total size, or -1 when storage did not say. + Size int64 +} + +var contentRangePattern = regexp.MustCompile(`^bytes (\d+)-(\d+)/(\d+|\*)$`) + +// GetObject makes one GET of a presigned storage URL, from byte offset on. +// Like PutObject it never sends the API key and never follows a redirect. +// Failures are *StorageError values; the caller decides whether to retry, +// since it knows how much it already wrote. +func (c *Client) GetObject(ctx context.Context, rawURL string, offset int64) (*Object, error) { + u, err := checkURL(rawURL) + if err != nil { + return nil, &StorageError{Op: OpDownloadArchive, Err: fmt.Errorf("refusing the download URL: %w", err)} + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, &StorageError{Op: OpDownloadArchive, Err: stripURL(err)} + } + req.Header.Set("User-Agent", c.userAgent) + if offset > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) + } + resp, err := c.storage.Do(req) + if err != nil { + return nil, &StorageError{Op: OpDownloadArchive, Err: fmt.Errorf("%s: %w", u.Host, stripURL(err))} + } + switch { + case resp.StatusCode == http.StatusOK: + return &Object{Body: resp.Body, Size: resp.ContentLength}, nil + case resp.StatusCode == http.StatusPartialContent && offset > 0: + m := contentRangePattern.FindStringSubmatch(resp.Header.Get("Content-Range")) + if m == nil || m[1] != strconv.FormatInt(offset, 10) { + resp.Body.Close() + return nil, &StorageError{Op: OpDownloadArchive, StatusCode: resp.StatusCode, Message: "storage answered with an unexpected range"} + } + size := int64(-1) + if total, err := strconv.ParseInt(m[3], 10, 64); err == nil { + size = total + } + return &Object{Body: resp.Body, Offset: offset, Size: size}, nil + } + defer resp.Body.Close() + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + return nil, &StorageError{Op: OpDownloadArchive, StatusCode: resp.StatusCode, Message: "storage answered with a redirect; refusing to follow it"} + } + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + e := parseStorageError(resp.StatusCode, body) + e.Op = OpDownloadArchive + return nil, e +} diff --git a/internal/api/exports_test.go b/internal/api/exports_test.go new file mode 100644 index 0000000..3e42f1c --- /dev/null +++ b/internal/api/exports_test.go @@ -0,0 +1,225 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Codeinwp/optimole-cli/internal/testserver" +) + +func TestExportDomainDecodesBothShapes(t *testing.T) { + var e Export + body := `{"id":7,"status":"running","domains":["example.com",{"domain":"directUpload","status":"completed","zip_files":[{"part_number":1,"url":"https://s3/x","size":"12"}]}]}` + if err := json.Unmarshal([]byte(body), &e); err != nil { + t.Fatal(err) + } + if len(e.Domains) != 2 || e.Domains[0].Domain != "example.com" || e.Domains[0].Status != "" { + t.Fatalf("name-only domain: %+v", e.Domains) + } + d := e.Domains[1] + if d.Domain != "directUpload" || d.Status != "completed" || len(d.Parts) != 1 || d.Parts[0].Number.Value != 1 || d.Parts[0].Size.Value != 12 { + t.Fatalf("domain object: %+v", d) + } + if err := json.Unmarshal([]byte(`{"domains":[42]}`), &e); err == nil { + t.Fatal("a number is neither a name nor a domain object") + } +} + +func TestIsFinalStatus(t *testing.T) { + for status, want := range map[string]bool{ + ExportQueued: false, ExportRunning: false, "": false, + ExportCompleted: true, ExportFailed: true, ExportCancelled: true, + } { + if got := IsFinalStatus(status); got != want { + t.Errorf("IsFinalStatus(%q) = %v", status, got) + } + } +} + +func TestCreateExportConflicts(t *testing.T) { + srv := testserver.New(t) + c, _ := newTestClient(t, srv.URL) + ctx := context.Background() + + exp, err := c.CreateExport(ctx) + if err != nil || exp.ID <= 0 || exp.Status != ExportQueued { + t.Fatalf("create: %+v, %v", exp, err) + } + _, err = c.CreateExport(ctx) + var apiErr *Error + if !errors.As(err, &apiErr) || apiErr.StatusCode != http.StatusConflict || apiErr.Code != CodeExportInProgress || apiErr.ExportID.Value != exp.ID { + t.Fatalf("in progress: %#v", err) + } + if !strings.Contains(err.Error(), "export_in_progress; export 1") { + t.Errorf("message: %v", err) + } + + srv2 := testserver.New(t) + for i := 0; i < testserver.ExportLimit; i++ { + srv2.AddExport(ExportCompleted) + } + c2, _ := newTestClient(t, srv2.URL) + _, err = c2.CreateExport(ctx) + if !errors.As(err, &apiErr) || apiErr.Code != CodeExportLimitReached || apiErr.Limit.Value != 5 || apiErr.Count.Value != 5 { + t.Fatalf("limit: %#v", err) + } + + srv2.LegacyExportConflicts = true + _, err = c2.CreateExport(ctx) + if !errors.As(err, &apiErr) || apiErr.StatusCode != http.StatusConflict || apiErr.Code != "" || apiErr.Limit.Valid { + t.Fatalf("legacy limit: %#v", err) + } +} + +func TestCreateExportIsNotRepeatedOnceItMayHaveStarted(t *testing.T) { + srv := testserver.New(t) + srv.Inject(testserver.RouteExportCreate, testserver.Response{CommitThenClose: true}) + c, _ := newTestClient(t, srv.URL) + _, err := c.CreateExport(context.Background()) + var netErr *NetworkError + if !errors.As(err, &netErr) || !netErr.MaybeCommitted { + t.Fatalf("got %v", err) + } + if n := srv.Count(testserver.RouteExportCreate); n != 1 { + t.Errorf("create requests: %d, want 1", n) + } + if ids := srv.ExportIDs(); len(ids) != 1 { + t.Errorf("exports: %v", ids) + } +} + +func TestDeleteExport(t *testing.T) { + srv := testserver.New(t) + c, _ := newTestClient(t, srv.URL) + ctx := context.Background() + done := srv.AddExport(ExportCompleted) + running := srv.AddExport(ExportRunning) + + var apiErr *Error + if err := c.DeleteExport(ctx, running); !errors.As(err, &apiErr) || apiErr.StatusCode != http.StatusConflict { + t.Fatalf("running export: %v", err) + } + if err := c.DeleteExport(ctx, 999); !errors.As(err, &apiErr) || apiErr.StatusCode != http.StatusNotFound { + t.Fatalf("unknown export: %v", err) + } + // The first DELETE works but its answer is lost; the retry's 404 means + // it is gone. + srv.Inject(testserver.RouteExportDelete, testserver.Response{CommitThenClose: true}) + if err := c.DeleteExport(ctx, done); err != nil { + t.Fatalf("delete after a lost answer: %v", err) + } + if ids := srv.ExportIDs(); len(ids) != 1 || ids[0] != running { + t.Errorf("exports left: %v", ids) + } +} + +func TestListExportsQuery(t *testing.T) { + var got string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.URL.RequestURI() + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"data":[{"id":3,"status":"completed","created_at":"2026-11-04T10:00:00.000000Z","domains":["a.com"]}],"meta":{"next_cursor":"abc","has_more":true}}`) + })) + defer ts.Close() + c, _ := newTestClient(t, ts.URL+"/api") + page, err := c.ListExports(context.Background(), ExportQuery{PerPage: 5, Cursor: "xyz"}) + if err != nil { + t.Fatal(err) + } + if got != "/api/asset-exports?cursor=xyz&per_page=5" { + t.Errorf("request: %s", got) + } + if len(page.Data) != 1 || page.Data[0].Domains[0].Domain != "a.com" || !page.Meta.HasMore || *page.Meta.NextCursor != "abc" { + t.Errorf("page: %+v", page) + } +} + +// archiveURL returns the URL of an export's first archive. +func archiveURL(t *testing.T, srv *testserver.Server, c *Client, data []byte) string { + t.Helper() + id := srv.AddExport(ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + exp, err := c.GetExport(context.Background(), id) + if err != nil { + t.Fatal(err) + } + return exp.Domains[0].Parts[0].URL +} + +func TestGetObjectRanges(t *testing.T) { + srv := testserver.New(t) + c, _ := newTestClient(t, srv.URL) + ctx := context.Background() + data := []byte("0123456789") + u := archiveURL(t, srv, c, data) + + read := func(offset int64) (*Object, string) { + t.Helper() + obj, err := c.GetObject(ctx, u, offset) + if err != nil { + t.Fatalf("offset %d: %v", offset, err) + } + defer obj.Body.Close() + b, _ := io.ReadAll(obj.Body) + return obj, string(b) + } + if obj, body := read(0); obj.Offset != 0 || obj.Size != 10 || body != "0123456789" { + t.Errorf("full: %+v %q", obj, body) + } + if obj, body := read(4); obj.Offset != 4 || obj.Size != 10 || body != "456789" { + t.Errorf("range: %+v %q", obj, body) + } + _, err := c.GetObject(ctx, u, 10) + var se *StorageError + if !errors.As(err, &se) || se.StatusCode != http.StatusRequestedRangeNotSatisfiable { + t.Fatalf("past the end: %v", err) + } + if !strings.HasPrefix(err.Error(), "download archive: ") || strings.Contains(err.Error(), "X-Amz-Signature") { + t.Errorf("message: %v", err) + } + + srv.Inject(testserver.RouteDownload, testserver.Response{IgnoreRange: true}) + if obj, body := read(4); obj.Offset != 0 || body != "0123456789" { + t.Errorf("range ignored: %+v %q", obj, body) + } + + srv.ExpireArchiveURLs() + if _, err := c.GetObject(ctx, u, 0); !errors.As(err, &se) || se.StatusCode != http.StatusForbidden || se.Code != "AccessDenied" { + t.Errorf("expired URL: %v", err) + } +} + +func TestGetObjectRefusals(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "" { + t.Error("storage GET carried an Authorization header") + } + switch r.URL.Path { + case "/redirect": + http.Redirect(w, r, "https://elsewhere.example/x", http.StatusFound) + case "/bad-range": + w.Header().Set("Content-Range", "bytes 0-9/10") + w.WriteHeader(http.StatusPartialContent) + io.WriteString(w, "0123456789") + } + })) + defer ts.Close() + c, _ := newTestClient(t, ts.URL+"/api") + ctx := context.Background() + + _, err := c.GetObject(ctx, ts.URL+"/redirect?X-Amz-Signature=secret", 0) + if err == nil || !strings.Contains(err.Error(), "redirect") || strings.Contains(err.Error(), "secret") { + t.Errorf("redirect: %v", err) + } + if _, err := c.GetObject(ctx, ts.URL+"/bad-range", 4); err == nil || !strings.Contains(err.Error(), "unexpected range") { + t.Errorf("range that starts elsewhere: %v", err) + } + if _, err := c.GetObject(ctx, "http://storage.example/x", 0); err == nil || !strings.Contains(err.Error(), "refusing the download URL") { + t.Errorf("plain http: %v", err) + } +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index f392ba0..fb7b47c 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -62,7 +62,7 @@ func DefaultEnv(build BuildInfo) Env { } } -const mainUsage = `optimole uploads local files into an Optimole media library. +const mainUsage = `optimole uploads local files into an Optimole media library and exports it. Usage: optimole [flags] [arguments] @@ -70,6 +70,7 @@ Usage: Commands: upload Upload files and directories into library folders folders List the library's folders + export Export the library as ZIP archives and download them login Save an API key (read from stdin) logout Delete the saved API key version Print the version @@ -89,6 +90,8 @@ func Run(ctx context.Context, args []string, env Env) int { return runUpload(ctx, rest, env) case "folders": return runFolders(ctx, rest, env) + case "export": + return runExport(ctx, rest, env) case "login": return runLogin(ctx, rest, env) case "logout": diff --git a/internal/cli/export.go b/internal/cli/export.go new file mode 100644 index 0000000..aeaca67 --- /dev/null +++ b/internal/cli/export.go @@ -0,0 +1,635 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net/http" + "path/filepath" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/Codeinwp/optimole-cli/internal/api" + "github.com/Codeinwp/optimole-cli/internal/export" + "github.com/Codeinwp/optimole-cli/internal/output" + "github.com/Codeinwp/optimole-cli/internal/upload" +) + +const exportUsage = `Usage: + optimole export [flags] [arguments] + +Exports the original files of the whole library (each whitelisted domain, plus +"directUpload" for the files uploaded straight to the library) as ZIP archives +of about 1 GB each, and downloads them. Exports need a full-access API key. + +Commands: + start Start an export, wait for it and download its archives + download [ID] Download an export's archives (default: the latest export), + waiting while it runs and resuming an earlier download + status [ID] Show an export's domains and archives (default: the latest) + list List the account's exports, newest first + delete ID Delete a finished export and its archives + +An account keeps at most 5 exports and runs one at a time. Finished exports are +deleted automatically 10 days after they finish. + +Run "optimole export -h" for a command's flags. +` + +const downloadFlagsUsage = ` -o, --output DIR Folder the export folder goes into (default: the current folder) + --domain NAME Only download this domain's archives (repeatable) + -c, --concurrency N Archives downloaded in parallel, 1-8 (default 2) + --timeout DURATION Give up after this long, e.g. 90m or 2h (default: no limit) + --json Print one JSON object per event and a final summary object +` + +const downloadNotes = ` +Ctrl-C and --timeout stop only the CLI: the export continues on the server, +and "optimole export download ID" picks up where the run stopped. + +Exit codes: 0 the export completed and every selected archive is downloaded, +1 the export failed or was cancelled, an archive failed, or the timeout passed, +2 usage or configuration error, 130 interrupted. +` + +const exportStartUsage = `Usage: + optimole export start [flags] + +Starts an export of the whole library, follows it, and downloads each archive +as soon as it is ready, to DIR/optimole-export-ID/DOMAIN/images-N.zip, with an +export.json describing them. If an export is already running, follows that +one instead. + +Flags: +` + downloadFlagsUsage + ` --no-wait Start the export and print its ID, without waiting +` + commonUsage + downloadNotes + +const exportDownloadUsage = `Usage: + optimole export download [ID|latest] [flags] + +Downloads the archives of an export (default: the latest) to +DIR/optimole-export-ID/DOMAIN/images-N.zip, with an export.json describing +them. While the export runs, each archive is downloaded as soon as it is ready. +Archives already downloaded are skipped and partial ones resumed. + +Flags: +` + downloadFlagsUsage + commonUsage + downloadNotes + +const exportStatusUsage = `Usage: + optimole export status [ID|latest] [--json] + +Shows an export's status and, for each domain, its status and finished +archives (default: the latest export). + +Flags: + --json Print the export as one JSON object (without download URLs) +` + commonUsage + +const exportListUsage = `Usage: + optimole export list [--json] + +Lists the account's exports, newest first. + +Flags: + --json Print one JSON object per export +` + commonUsage + +const exportDeleteUsage = `Usage: + optimole export delete ID + +Deletes a finished export and its archives on the server, which frees one of +the account's 5 export slots. Downloaded files are not touched. +` + commonUsage + +const ( + // exportLimit is how many exports the dashboard lets an account keep. + exportLimit = 5 + // exportsPerPage is more than an account keeps, so one page lists them. + exportsPerPage = 20 + maxExportPages = 100 +) + +func runExport(ctx context.Context, args []string, env Env) int { + if len(args) == 0 { + fmt.Fprint(env.Stderr, exportUsage) + return ExitUsage + } + cmd, rest := args[0], args[1:] + switch cmd { + case "start": + return runExportStart(ctx, rest, env) + case "download": + return runExportDownload(ctx, rest, env) + case "status": + return runExportStatus(ctx, rest, env) + case "list": + return runExportList(ctx, rest, env) + case "delete": + return runExportDelete(ctx, rest, env) + case "help", "-h", "-help", "--help": + fmt.Fprint(env.Stdout, exportUsage) + return ExitOK + default: + fmt.Fprintf(env.Stderr, "optimole export: unknown command %q\n\n%s", cmd, exportUsage) + return ExitUsage + } +} + +// downloadFlags are the flags of "export start" and "export download". +type downloadFlags struct { + common commonFlags + output string + domains stringList + concurrency int + timeout time.Duration + jsonOut bool +} + +func (d *downloadFlags) register(fs *flag.FlagSet) { + d.common.register(fs, true) + fs.StringVar(&d.output, "o", ".", "") + fs.StringVar(&d.output, "output", ".", "") + fs.Var(&d.domains, "domain", "") + fs.IntVar(&d.concurrency, "c", export.DefaultConcurrency, "") + fs.IntVar(&d.concurrency, "concurrency", export.DefaultConcurrency, "") + fs.DurationVar(&d.timeout, "timeout", 0, "") + fs.BoolVar(&d.jsonOut, "json", false, "") +} + +// check validates the flags; it returns a usage problem or "". +func (d *downloadFlags) check() string { + switch { + case d.concurrency < 1 || d.concurrency > export.MaxConcurrency: + return fmt.Sprintf("--concurrency must be between 1 and %d", export.MaxConcurrency) + case d.timeout < 0: + return "--timeout must not be negative" + case strings.TrimSpace(d.output) == "": + return "--output must not be empty" + } + for _, dom := range d.domains { + if strings.TrimSpace(dom) == "" { + return "--domain must not be empty" + } + } + return "" +} + +func runExportStart(ctx context.Context, args []string, env Env) int { + const name = "export start" + fs := newFlagSet(name, exportStartUsage, env) + var df downloadFlags + df.register(fs) + var noWait bool + fs.BoolVar(&noWait, "no-wait", false, "") + rest, err := parseArgs(fs, args) + if err != nil { + return flagExit(err) + } + if len(rest) > 0 { + return usageError(env, name, "takes no arguments") + } + if msg := df.check(); msg != "" { + return usageError(env, name, "%s", msg) + } + sess, err := df.common.connect(env) + if err != nil { + fmt.Fprintf(env.Stderr, "optimole %s: %v\n", name, err) + return ExitUsage + } + runCtx, cancel := withTimeout(ctx, df.timeout) + defer cancel() + + id, created, code := startExport(runCtx, env, sess) + if code != ExitOK { + return code + } + rep := output.NewExportReporter(env.Stdout, df.jsonOut, id, sess.redact.String) + rep.Started(created) + if noWait { + if rep.Err() != nil { + return outputFailed(env, rep.Err()) + } + return ExitOK + } + return downloadExport(ctx, runCtx, env, sess, rep, name, id, &df) +} + +// startExport creates an export, or finds the one already running. It +// returns the export ID, whether this call created it, and an exit code. +func startExport(ctx context.Context, env Env, sess *session) (int64, bool, int) { + const name = "export start" + exp, err := sess.client.CreateExport(ctx) + if err == nil { + return exp.ID, true, ExitOK + } + var apiErr *api.Error + var netErr *api.NetworkError + switch { + case errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusConflict: + if apiErr.Code == api.CodeExportInProgress && apiErr.ExportID.Valid { + return apiErr.ExportID.Value, false, ExitOK + } + // Servers without error codes answer both conflicts with a bare 409: + // the list says which one it is. + exports, listErr := listAllExports(ctx, sess.client) + if listErr != nil { + return 0, false, exportError(env, sess, name, err) + } + if len(exports) > 0 && !api.IsFinalStatus(exports[0].Status) { + return exports[0].ID, false, ExitOK + } + fmt.Fprintf(env.Stderr, "optimole %s: %s\n", name, sess.redact.Error(err)) + if apiErr.Code == api.CodeExportLimitReached || len(exports) >= exportLimit { + printLimitHint(env, exports) + } + return 0, false, ExitFailure + case errors.As(err, &netErr) && netErr.MaybeCommitted: + // The export may have started: follow it if one runs. + exports, listErr := listAllExports(ctx, sess.client) + if listErr == nil && len(exports) > 0 && !api.IsFinalStatus(exports[0].Status) { + return exports[0].ID, false, ExitOK + } + } + return 0, false, exportError(env, sess, name, err) +} + +func printLimitHint(env Env, exports []api.Export) { + hint := `Delete one with "optimole export delete ID"; finished exports are also deleted 10 days after they finish.` + for i := len(exports) - 1; i >= 0; i-- { + if api.IsFinalStatus(exports[i].Status) { + e := exports[i] + hint += fmt.Sprintf(" Oldest: export %d (%s, %s).", e.ID, e.Status, displayTime(e.CreatedAt)) + break + } + } + fmt.Fprintln(env.Stderr, hint) +} + +func runExportDownload(ctx context.Context, args []string, env Env) int { + const name = "export download" + fs := newFlagSet(name, exportDownloadUsage, env) + var df downloadFlags + df.register(fs) + rest, err := parseArgs(fs, args) + if err != nil { + return flagExit(err) + } + if len(rest) > 1 { + return usageError(env, name, "takes at most one export ID") + } + if msg := df.check(); msg != "" { + return usageError(env, name, "%s", msg) + } + ref := "latest" + if len(rest) == 1 { + ref = rest[0] + } + if _, err := parseExportRef(ref); err != nil { + return usageError(env, name, "%v", err) + } + sess, err := df.common.connect(env) + if err != nil { + fmt.Fprintf(env.Stderr, "optimole %s: %v\n", name, err) + return ExitUsage + } + runCtx, cancel := withTimeout(ctx, df.timeout) + defer cancel() + id, code := resolveExport(runCtx, env, sess, name, ref) + if code != ExitOK { + return code + } + rep := output.NewExportReporter(env.Stdout, df.jsonOut, id, sess.redact.String) + return downloadExport(ctx, runCtx, env, sess, rep, name, id, &df) +} + +// downloadExport runs the download of export id. ctx is the command's own +// context (Ctrl-C), runCtx the one --timeout bounds. +func downloadExport(ctx, runCtx context.Context, env Env, sess *session, rep *output.ExportReporter, name string, id int64, df *downloadFlags) int { + dir := filepath.Join(df.output, fmt.Sprintf("optimole-export-%d", id)) + out := export.Run(runCtx, export.Options{ + Client: sess.client, + ExportID: id, + Dir: dir, + Domains: df.domains, + Concurrency: df.concurrency, + Sleep: env.Sleep, + OnEvent: rep.Event, + }) + var unknown *export.UnknownDomainError + if errors.As(out.Err, &unknown) { + return usageError(env, name, "--domain: %v", unknown) + } + interrupted := ctx.Err() != nil + timedOut := !interrupted && errors.Is(out.Err, context.DeadlineExceeded) + if out.Export != nil { + rep.Finish(out, dir, interrupted, timedOut) + } + if interrupted || timedOut { + resume := fmt.Sprintf("optimole export download %d", id) + if df.output != "." { + resume += " -o " + strconv.Quote(df.output) + } + fmt.Fprintf(env.Stderr, "Export %d continues on the server. Resume with: %s\n", id, resume) + } + switch { + case interrupted: + return ExitInterrupted + case timedOut: + fmt.Fprintf(env.Stderr, "optimole %s: timed out after %s\n", name, df.timeout) + return ExitFailure + case out.Err != nil: + return exportError(env, sess, name, out.Err) + case rep.Err() != nil: + return outputFailed(env, rep.Err()) + case out.Failed > 0 || out.Export.Status != api.ExportCompleted: + return ExitFailure + } + return ExitOK +} + +func runExportStatus(ctx context.Context, args []string, env Env) int { + const name = "export status" + fs := newFlagSet(name, exportStatusUsage, env) + var common commonFlags + common.register(fs, true) + var jsonOut bool + fs.BoolVar(&jsonOut, "json", false, "") + rest, err := parseArgs(fs, args) + if err != nil { + return flagExit(err) + } + if len(rest) > 1 { + return usageError(env, name, "takes at most one export ID") + } + ref := "latest" + if len(rest) == 1 { + ref = rest[0] + } + if _, err := parseExportRef(ref); err != nil { + return usageError(env, name, "%v", err) + } + sess, err := common.connect(env) + if err != nil { + fmt.Fprintf(env.Stderr, "optimole %s: %v\n", name, err) + return ExitUsage + } + id, code := resolveExport(ctx, env, sess, name, ref) + if code != ExitOK { + return code + } + exp, err := sess.client.GetExport(ctx, id) + if err != nil { + return exportError(env, sess, name, err) + } + + out := output.NewErrWriter(env.Stdout) + if jsonOut { + enc := json.NewEncoder(out) + enc.SetEscapeHTML(false) + enc.Encode(statusJSON(exp)) + } else { + fmt.Fprintf(out, "Export %d: %s, started %s\n", exp.ID, exp.Status, displayTime(exp.CreatedAt)) + tw := tabwriter.NewWriter(out, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "DOMAIN\tSTATUS\tARCHIVES\tSIZE") + for _, d := range exp.Domains { + var total int64 + for _, p := range d.Parts { + total += p.Size.Value + } + size := "-" + if len(d.Parts) > 0 { + size = upload.HumanSize(total) + } + fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n", d.Domain, d.Status, len(d.Parts), size) + } + tw.Flush() + } + if err := out.Err(); err != nil { + return outputFailed(env, err) + } + return ExitOK +} + +// exportStatusLine is the JSON of "export status": the export without URLs. +type exportStatusLine struct { + ID int64 `json:"id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + Domains []domainStatusLine `json:"domains"` +} + +type domainStatusLine struct { + Domain string `json:"domain"` + Status string `json:"status"` + Archives []archiveStatusLine `json:"archives"` +} + +type archiveStatusLine struct { + PartNumber int64 `json:"part_number"` + Size int64 `json:"size"` +} + +func statusJSON(exp *api.Export) exportStatusLine { + l := exportStatusLine{ID: exp.ID, Status: exp.Status, CreatedAt: exp.CreatedAt, Domains: []domainStatusLine{}} + for _, d := range exp.Domains { + dl := domainStatusLine{Domain: d.Domain, Status: d.Status, Archives: []archiveStatusLine{}} + for _, p := range d.Parts { + dl.Archives = append(dl.Archives, archiveStatusLine{PartNumber: p.Number.Value, Size: p.Size.Value}) + } + l.Domains = append(l.Domains, dl) + } + return l +} + +func runExportList(ctx context.Context, args []string, env Env) int { + const name = "export list" + fs := newFlagSet(name, exportListUsage, env) + var common commonFlags + common.register(fs, true) + var jsonOut bool + fs.BoolVar(&jsonOut, "json", false, "") + rest, err := parseArgs(fs, args) + if err != nil { + return flagExit(err) + } + if len(rest) > 0 { + return usageError(env, name, "takes no arguments") + } + sess, err := common.connect(env) + if err != nil { + fmt.Fprintf(env.Stderr, "optimole %s: %v\n", name, err) + return ExitUsage + } + exports, err := listAllExports(ctx, sess.client) + if err != nil { + return exportError(env, sess, name, err) + } + + out := output.NewErrWriter(env.Stdout) + switch { + case jsonOut: + enc := json.NewEncoder(out) + enc.SetEscapeHTML(false) + for _, e := range exports { + names := make([]string, 0, len(e.Domains)) + for _, d := range e.Domains { + names = append(names, d.Domain) + } + enc.Encode(struct { + ID int64 `json:"id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + Domains []string `json:"domains"` + }{e.ID, e.Status, e.CreatedAt, names}) + } + case len(exports) == 0: + fmt.Fprintln(out, `No exports. Start one with "optimole export start".`) + default: + tw := tabwriter.NewWriter(out, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tSTATUS\tCREATED\tDOMAINS") + for _, e := range exports { + fmt.Fprintf(tw, "%d\t%s\t%s\t%d\n", e.ID, e.Status, displayTime(e.CreatedAt), len(e.Domains)) + } + tw.Flush() + } + if err := out.Err(); err != nil { + return outputFailed(env, err) + } + return ExitOK +} + +func runExportDelete(ctx context.Context, args []string, env Env) int { + const name = "export delete" + fs := newFlagSet(name, exportDeleteUsage, env) + var common commonFlags + common.register(fs, true) + rest, err := parseArgs(fs, args) + if err != nil { + return flagExit(err) + } + if len(rest) != 1 { + return usageError(env, name, "takes one export ID") + } + id, err := parseExportRef(rest[0]) + if err != nil || id == 0 { + return usageError(env, name, "%q is not an export ID", rest[0]) + } + sess, err := common.connect(env) + if err != nil { + fmt.Fprintf(env.Stderr, "optimole %s: %v\n", name, err) + return ExitUsage + } + if err := sess.client.DeleteExport(ctx, id); err != nil { + var apiErr *api.Error + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusConflict { + fmt.Fprintf(env.Stderr, "optimole %s: %s\nOnly finished exports can be deleted; this one is still queued or running.\n", name, sess.redact.Error(err)) + return ExitFailure + } + return exportError(env, sess, name, err) + } + out := output.NewErrWriter(env.Stdout) + fmt.Fprintf(out, "Deleted export %d.\n", id) + if err := out.Err(); err != nil { + return outputFailed(env, err) + } + return ExitOK +} + +// parseExportRef reads an export ID; "latest" gives 0. +func parseExportRef(ref string) (int64, error) { + if strings.EqualFold(ref, "latest") { + return 0, nil + } + id, err := strconv.ParseInt(ref, 10, 64) + if err != nil || id <= 0 { + return 0, fmt.Errorf("%q is not an export ID (or \"latest\")", ref) + } + return id, nil +} + +// resolveExport turns an export reference into an ID, looking up the latest +// export when asked. +func resolveExport(ctx context.Context, env Env, sess *session, name, ref string) (int64, int) { + id, _ := parseExportRef(ref) + if id > 0 { + return id, ExitOK + } + page, err := sess.client.ListExports(ctx, api.ExportQuery{PerPage: 1}) + if err != nil { + return 0, exportError(env, sess, name, err) + } + if len(page.Data) == 0 { + fmt.Fprintf(env.Stderr, "optimole %s: there are no exports yet; start one with \"optimole export start\"\n", name) + return 0, ExitFailure + } + return page.Data[0].ID, ExitOK +} + +// listAllExports returns every export of the account, newest first. +func listAllExports(ctx context.Context, client *api.Client) ([]api.Export, error) { + var all []api.Export + q := api.ExportQuery{PerPage: exportsPerPage} + seen := map[string]bool{} + for page := 0; page < maxExportPages; page++ { + res, err := client.ListExports(ctx, q) + if err != nil { + return nil, err + } + all = append(all, res.Data...) + next := "" + if res.Meta.NextCursor != nil { + next = *res.Meta.NextCursor + } + if !res.Meta.HasMore || next == "" || seen[next] { + break + } + seen[next] = true + q.Cursor = next + } + return all, nil +} + +// exportError prints a failed export call and returns its exit code. +func exportError(env Env, sess *session, name string, err error) int { + msg := sess.redact.Error(err) + var apiErr *api.Error + switch { + case errors.Is(err, context.Canceled): + fmt.Fprintf(env.Stderr, "optimole %s: interrupted\n", name) + return ExitInterrupted + case isRejectedKey(err): + fmt.Fprintf(env.Stderr, "optimole %s: %s\nThe API key (from %s) was rejected.\n", name, msg, sess.settings.KeySource) + case errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusForbidden && isScopedKey(sess.settings.APIKey): + fmt.Fprintf(env.Stderr, "optimole %s: %s\nExports need a full-access API key; folder-scoped keys cannot export.\n", name, msg) + case errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound && apiErr.Op == api.OpGetExport: + fmt.Fprintf(env.Stderr, "optimole %s: %s\nNo such export: finished exports are deleted 10 days after they finish (\"optimole export list\" shows the others).\n", name, msg) + default: + fmt.Fprintf(env.Stderr, "optimole %s: %s\n", name, msg) + } + return ExitFailure +} + +// withTimeout bounds ctx by d when d is set. +func withTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { + if d <= 0 { + return context.WithCancel(ctx) + } + return context.WithTimeout(ctx, d) +} + +// displayTime shows an API timestamp in UTC to the minute, or as given when +// it does not parse. +func displayTime(ts string) string { + t, err := time.Parse(time.RFC3339Nano, ts) + if err != nil { + if ts == "" { + return "-" + } + return ts + } + return t.UTC().Format("2006-01-02 15:04 UTC") +} diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go new file mode 100644 index 0000000..2545017 --- /dev/null +++ b/internal/cli/export_test.go @@ -0,0 +1,358 @@ +package cli + +import ( + "archive/zip" + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Codeinwp/optimole-cli/internal/config" + "github.com/Codeinwp/optimole-cli/internal/testserver" +) + +func archive(t *testing.T, names ...string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, n := range names { + w, _ := zw.Create(n) + w.Write([]byte("data of " + n)) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func (h *harness) twoDomains() { + h.srv.SetExportDomains( + testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{archive(h.t, "a.jpg"), archive(h.t, "b.jpg")}}, + testserver.ExportDomain{Domain: "directUpload", Archives: [][]byte{archive(h.t, "c.png")}}, + ) +} + +func TestExportStartDownloadsEverything(t *testing.T) { + h := newHarness(t).withKey() + h.twoDomains() + out := t.TempDir() + if code := h.run("export", "start", "-o", out); code != ExitOK { + t.Fatalf("exit %d, stderr %s", code, h.stderr.String()) + } + got := h.stdout.String() + for _, want := range []string{ + "Started export 1.\n", + "Export 1: running\n", + " example.com: completed, 2 archives\n", + "downloaded " + filepath.Join(out, "optimole-export-1", "example.com", "images-2.zip"), + "Export 1: completed\n", + "Export 1 completed: 3 downloaded, 0 already there, 0 failed", + } { + if !strings.Contains(got, want) { + t.Errorf("stdout lacks %q:\n%s", want, got) + } + } + for _, rel := range []string{"example.com/images-1.zip", "example.com/images-2.zip", "directUpload/images-1.zip", "export.json"} { + if _, err := os.Stat(filepath.Join(out, "optimole-export-1", filepath.FromSlash(rel))); err != nil { + t.Error(err) + } + } + + // A second run finds everything in place. + if code := h.run("export", "download", "1", "-o", out); code != ExitOK { + t.Fatalf("download exit %d", code) + } + if !strings.Contains(h.stdout.String(), "0 downloaded, 3 already there") { + t.Errorf("second run: %s", h.stdout.String()) + } +} + +func TestExportStartJSON(t *testing.T) { + h := newHarness(t).withKey() + h.twoDomains() + out := t.TempDir() + if code := h.run("export", "start", "--json", "--output", out, "--domain", "example.com"); code != ExitOK { + t.Fatalf("exit %d, stderr %s", code, h.stderr.String()) + } + if strings.Contains(h.stdout.String(), "X-Amz-Signature") || strings.Contains(h.stdout.String(), "http") { + t.Errorf("output holds a URL: %s", h.stdout.String()) + } + lines := h.jsonLines() + if lines[0]["event"] != "started" || lines[0]["created"] != true || lines[0]["id"] != float64(1) { + t.Errorf("first line: %v", lines[0]) + } + archives := 0 + for _, l := range lines { + if l["event"] == "archive" { + archives++ + if l["status"] != "downloaded" || l["domain"] != "example.com" || l["size"] == nil || !strings.HasPrefix(l["path"].(string), out) { + t.Errorf("archive line: %v", l) + } + } + if l["domain"] == "directUpload" { + t.Errorf("unselected domain reported: %v", l) + } + } + sum, _ := lines[len(lines)-1]["summary"].(map[string]any) + if archives != 2 || sum["status"] != "completed" || sum["downloaded"] != float64(2) || sum["dir"] != filepath.Join(out, "optimole-export-1") { + t.Errorf("archives %d, summary %v", archives, sum) + } +} + +func TestExportStartFollowsTheRunningExport(t *testing.T) { + for _, legacy := range []bool{false, true} { + h := newHarness(t).withKey() + h.srv.LegacyExportConflicts = legacy + h.srv.AddExport("completed") + running := h.srv.AddExport("running", testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{archive(t, "a")}}) + out := t.TempDir() + if code := h.run("export", "start", "-o", out); code != ExitOK { + t.Fatalf("legacy=%v: exit %d, stderr %s", legacy, code, h.stderr.String()) + } + if !strings.Contains(h.stdout.String(), "Export 2 is already running; following it.") { + t.Errorf("legacy=%v: %s", legacy, h.stdout.String()) + } + if _, err := os.Stat(filepath.Join(out, "optimole-export-2", "example.com", "images-1.zip")); err != nil { + t.Errorf("legacy=%v: %v", legacy, err) + } + if ids := h.srv.ExportIDs(); len(ids) != 2 || ids[1] != running { + t.Errorf("legacy=%v: exports %v", legacy, ids) + } + // The coded answer names the export; only older servers need the list. + if lists := h.srv.Count(testserver.RouteExportList); (lists == 0) == legacy { + t.Errorf("legacy=%v: %d list requests", legacy, lists) + } + } +} + +func TestExportStartAtTheLimit(t *testing.T) { + for _, legacy := range []bool{false, true} { + h := newHarness(t).withKey() + h.srv.LegacyExportConflicts = legacy + for i := 0; i < testserver.ExportLimit; i++ { + h.srv.AddExport("completed") + } + if code := h.run("export", "start"); code != ExitFailure { + t.Fatalf("legacy=%v: exit %d", legacy, code) + } + stderr := h.stderr.String() + for _, want := range []string{"Maximum asset export count reached", `"optimole export delete ID"`, "Oldest: export 1 (completed, 2026-11-04 10:01 UTC)."} { + if !strings.Contains(stderr, want) { + t.Errorf("legacy=%v: stderr lacks %q:\n%s", legacy, want, stderr) + } + } + if !legacy && !strings.Contains(stderr, "export_limit_reached; limit 5, used 5") { + t.Errorf("no code: %s", stderr) + } + } +} + +func TestExportStartNoWait(t *testing.T) { + h := newHarness(t).withKey() + h.twoDomains() + if code := h.run("export", "start", "--no-wait"); code != ExitOK { + t.Fatalf("exit %d", code) + } + if h.stdout.String() != "Started export 1.\n" || h.srv.Count(testserver.RouteExportGet) != 0 { + t.Errorf("stdout %q, %d looks at the export", h.stdout.String(), h.srv.Count(testserver.RouteExportGet)) + } +} + +func TestExportStartAfterALostAnswer(t *testing.T) { + h := newHarness(t).withKey() + h.twoDomains() + h.srv.Inject(testserver.RouteExportCreate, testserver.Response{CommitThenClose: true}) + if code := h.run("export", "start", "-o", t.TempDir()); code != ExitOK { + t.Fatalf("exit %d, stderr %s", code, h.stderr.String()) + } + if n := len(h.srv.ExportIDs()); n != 1 || h.srv.Count(testserver.RouteExportCreate) != 1 { + t.Errorf("exports %d, create requests %d", n, h.srv.Count(testserver.RouteExportCreate)) + } +} + +func TestExportDownloadFailedExport(t *testing.T) { + h := newHarness(t).withKey() + h.srv.AddExport("failed", + testserver.ExportDomain{Domain: "broken.com", Fail: true}, + testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{archive(t, "a")}}, + ) + if code := h.run("export", "download", "-o", t.TempDir()); code != ExitFailure { + t.Fatalf("exit %d", code) + } + if !strings.Contains(h.stdout.String(), "Export 1 failed (failed: broken.com): 1 downloaded") { + t.Errorf("stdout: %s", h.stdout.String()) + } +} + +func TestExportDownloadStopsAndTellsHowToResume(t *testing.T) { + h := newHarness(t).withKey() + h.srv.PauseExports(true) + h.srv.AddExport("running", testserver.ExportDomain{Domain: "example.com"}) + out := filepath.Join(t.TempDir(), "my exports") + + if code := h.run("export", "download", "latest", "-o", out, "--timeout", "150ms"); code != ExitFailure { + t.Fatalf("timeout: exit %d", code) + } + resume := `Resume with: optimole export download 1 -o "` + out + `"` + if !strings.Contains(h.stderr.String(), "timed out after 150ms") || !strings.Contains(h.stderr.String(), resume) { + t.Errorf("timeout stderr: %s", h.stderr.String()) + } + if !strings.Contains(h.stdout.String(), "Export 1 running: 0 downloaded") || !strings.Contains(h.stdout.String(), "Timed out.") { + t.Errorf("timeout stdout: %s", h.stdout.String()) + } + + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(100*time.Millisecond, cancel) + if code := h.runCtx(ctx, "export", "download", "1", "-o", out); code != ExitInterrupted { + t.Fatalf("interrupt: exit %d", code) + } + if !strings.Contains(h.stderr.String(), resume) || !strings.Contains(h.stdout.String(), "Interrupted.") { + t.Errorf("interrupt: stdout %s stderr %s", h.stdout.String(), h.stderr.String()) + } +} + +func TestExportUsageErrors(t *testing.T) { + h := newHarness(t).withKey() + for _, args := range [][]string{ + {"export"}, + {"export", "nope"}, + {"export", "start", "extra"}, + {"export", "start", "-c", "9"}, + {"export", "start", "--timeout", "-1s"}, + {"export", "start", "--domain", " "}, + {"export", "download", "abc"}, + {"export", "download", "1", "2"}, + {"export", "status", "0"}, + {"export", "list", "x"}, + {"export", "delete"}, + {"export", "delete", "latest"}, + } { + if code := h.run(args...); code != ExitUsage { + t.Errorf("%v: exit %d", args, code) + } + } + if h.srv.Count(testserver.RouteExportCreate)+h.srv.Count(testserver.RouteExportList)+h.srv.Count(testserver.RouteExportDelete) != 0 { + t.Error("a usage error reached the server") + } + + h.srv.AddExport("completed", testserver.ExportDomain{Domain: "example.com"}) + if code := h.run("export", "download", "1", "--domain", "other.com", "-o", t.TempDir()); code != ExitUsage { + t.Errorf("unknown domain: exit %d", code) + } + if !strings.Contains(h.stderr.String(), `export has no domain "other.com" (it has: example.com)`) { + t.Errorf("unknown domain: %s", h.stderr.String()) + } +} + +func TestExportHelp(t *testing.T) { + h := newHarness(t) + for _, args := range [][]string{{"help", "export"}, {"export", "help"}} { + if code := h.run(args...); code != ExitOK { + t.Errorf("%v: exit %d", args, code) + } + if !strings.Contains(h.stdout.String()+h.stderr.String(), "optimole export ") { + t.Errorf("%v: no usage", args) + } + } + if code := h.run("export", "start", "-h"); code != ExitOK || !strings.Contains(h.stderr.String(), "--no-wait") { + t.Errorf("start -h: exit %d", code) + } +} + +func TestExportStatus(t *testing.T) { + h := newHarness(t).withKey() + h.srv.AddExport("completed", + testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{make([]byte, 2048), make([]byte, 1024)}}, + testserver.ExportDomain{Domain: "directUpload"}, + ) + if code := h.run("export", "status"); code != ExitOK { + t.Fatalf("exit %d: %s", code, h.stderr.String()) + } + got := h.stdout.String() + for _, want := range []string{"Export 1: completed, started 2026-11-04 10:01 UTC", "example.com completed 2 3.0 KB", "directUpload completed 0 -"} { + if !strings.Contains(got, want) { + t.Errorf("stdout lacks %q:\n%s", want, got) + } + } + if code := h.run("export", "status", "1", "--json"); code != ExitOK { + t.Fatalf("json exit %d", code) + } + if strings.Contains(h.stdout.String(), "http") { + t.Errorf("status JSON holds a URL: %s", h.stdout.String()) + } + l := h.jsonLines() + domains := l[0]["domains"].([]any) + first := domains[0].(map[string]any)["archives"].([]any)[0].(map[string]any) + if l[0]["id"] != float64(1) || first["part_number"] != float64(1) || first["size"] != float64(2048) { + t.Errorf("status JSON: %v", l[0]) + } +} + +func TestExportList(t *testing.T) { + h := newHarness(t).withKey() + if code := h.run("export", "list"); code != ExitOK || !strings.Contains(h.stdout.String(), "No exports.") { + t.Fatalf("empty list: exit %d %q", code, h.stdout.String()) + } + if code := h.run("export", "download"); code != ExitFailure || !strings.Contains(h.stderr.String(), "there are no exports yet") { + t.Errorf("latest of none: exit %d %s", code, h.stderr.String()) + } + h.srv.AddExport("completed", testserver.ExportDomain{Domain: "a.com"}, testserver.ExportDomain{Domain: "b.com"}) + h.srv.AddExport("running") + if code := h.run("export", "list"); code != ExitOK { + t.Fatalf("exit %d", code) + } + lines := strings.Split(strings.TrimSpace(h.stdout.String()), "\n") + if len(lines) != 3 || !strings.HasPrefix(lines[1], "2 running") || !strings.HasPrefix(lines[2], "1 completed 2026-11-04 10:01 UTC 2") { + t.Errorf("list:\n%s", h.stdout.String()) + } + if code := h.run("export", "list", "--json"); code != ExitOK { + t.Fatalf("json exit %d", code) + } + if l := h.jsonLines(); len(l) != 2 || l[1]["id"] != float64(1) || len(l[1]["domains"].([]any)) != 2 { + t.Errorf("list JSON: %v", l) + } +} + +func TestExportDelete(t *testing.T) { + h := newHarness(t).withKey() + done := h.srv.AddExport("completed") + h.srv.AddExport("running") + if code := h.run("export", "delete", "1"); code != ExitOK || h.stdout.String() != "Deleted export 1.\n" { + t.Fatalf("delete: exit %d %q", code, h.stdout.String()) + } + if code := h.run("export", "delete", "2"); code != ExitFailure || !strings.Contains(h.stderr.String(), "Only finished exports can be deleted") { + t.Errorf("running: exit %d %s", code, h.stderr.String()) + } + if code := h.run("export", "delete", "1"); code != ExitFailure || !strings.Contains(h.stderr.String(), "HTTP 404") { + t.Errorf("deleted twice: exit %d %s", code, h.stderr.String()) + } + if ids := h.srv.ExportIDs(); len(ids) != 1 || ids[0] == done { + t.Errorf("exports left: %v", ids) + } +} + +func TestExportNeedsAFullAccessKey(t *testing.T) { + h := newHarness(t) + const scoped = "optml-s-scoped-key-0123456789" + h.srv.AddKey(scoped, testserver.Key{Scoped: true}) + h.vars[config.EnvAPIKey] = scoped + if code := h.run("export", "start"); code != ExitFailure { + t.Fatalf("exit %d", code) + } + if !strings.Contains(h.stderr.String(), "Exports need a full-access API key") || strings.Contains(h.stderr.String(), scoped) { + t.Errorf("stderr: %s", h.stderr.String()) + } +} + +func TestExportMissingExport(t *testing.T) { + h := newHarness(t).withKey() + if code := h.run("export", "download", "99", "-o", t.TempDir()); code != ExitFailure { + t.Fatalf("exit %d", code) + } + if !strings.Contains(h.stderr.String(), "No such export: finished exports are deleted 10 days after they finish") { + t.Errorf("stderr: %s", h.stderr.String()) + } +} diff --git a/internal/export/download.go b/internal/export/download.go new file mode 100644 index 0000000..d68d41b --- /dev/null +++ b/internal/export/download.go @@ -0,0 +1,315 @@ +package export + +import ( + "archive/zip" + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/Codeinwp/optimole-cli/internal/api" +) + +const ( + partSuffix = ".part" + // maxResumes bounds the attempts of one archive even while each attempt + // makes progress. + maxResumes = 50 + maxBackoff = 30 * time.Second +) + +// stallTimeout aborts an attempt whose body stops arriving (tests shorten it). +var stallTimeout = 2 * time.Minute + +// errNeedURL means storage refused the URL (it expired) or the archive is not +// the size the export listed: a fresh look at the export gives the current +// URL and size. +var errNeedURL = errors.New("the download URL was refused or the archive changed") + +// errStalled is an attempt aborted because no bytes arrived for stallTimeout. +var errStalled = errors.New("the download stalled") + +// writeError is a failure writing to the local disk; retrying will not help. +type writeError struct{ err error } + +func (e *writeError) Error() string { return e.err.Error() } +func (e *writeError) Unwrap() error { return e.err } + +// job is one archive to download. +type job struct { + path string + url string + // size is the size the export listed, or -1 when it gave none. + size int64 +} + +// jobResult is how a job ended. +type jobResult struct { + // present: the archive was already on disk, nothing was downloaded. + present bool + // written counts the bytes written to disk, failed attempts included. + written int64 + // size is the archive's size once it is in place. + size int64 + err error +} + +// onDisk reports whether the archive is already in place: a file of the +// listed size (only checked archives are renamed into place). +func onDisk(j job) bool { + if j.size < 0 { + return false + } + st, err := os.Stat(j.path) + return err == nil && st.Mode().IsRegular() && st.Size() == j.size +} + +// download puts one archive at j.path. The bytes go to a ".part" file named +// after the expected size, so a later run resumes only an archive of the same +// size; the file is renamed into place once its size matches and it opens as +// a ZIP. A broken ZIP is downloaded once more from scratch. +func (r *runner) download(ctx context.Context, j job) jobResult { + if err := os.MkdirAll(filepath.Dir(j.path), 0o755); err != nil { + return jobResult{err: &writeError{err}} + } + tmp := tempPath(j) + removeStaleTemps(j.path, tmp) + + var res jobResult + for round := 0; round < 2; round++ { + if err := r.fetch(ctx, j, tmp, &res); err != nil { + res.err = err + return res + } + size, err := checkArchive(tmp, j.size) + if err == nil { + if err := os.Rename(tmp, j.path); err != nil { + res.err = &writeError{err} + return res + } + return jobResult{written: res.written, size: size} + } + os.Remove(tmp) + res.err = err + } + return res +} + +// tempPath names the partial file after the size it should reach. +func tempPath(j job) string { + if j.size < 0 { + return j.path + partSuffix + } + return j.path + "." + strconv.FormatInt(j.size, 10) + partSuffix +} + +// removeStaleTemps deletes partial files an earlier run left for another +// version (size) of the archive. +func removeStaleTemps(final, keep string) { + dir, base := filepath.Split(final) + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, e := range entries { + name := e.Name() + if strings.HasPrefix(name, base+".") && strings.HasSuffix(name, partSuffix) && filepath.Join(dir, name) != keep { + os.Remove(filepath.Join(dir, name)) + } + } +} + +// fetch completes tmp from storage, resuming from its current size. Network +// failures, 5xx answers, stalls and early ends are retried with backoff; the +// count restarts whenever an attempt makes progress. +func (r *runner) fetch(ctx context.Context, j job, tmp string, res *jobResult) error { + failures := 0 + for attempt := 1; ; attempt++ { + n, err := r.fetchOnce(ctx, j, tmp) + res.written += n + if err == nil { + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + if !retryable(err) || attempt >= maxResumes { + return err + } + if n > 0 { + failures = 0 + } + failures++ + if failures >= api.MaxAttempts { + return err + } + if err := r.sleep(ctx, backoff(failures)); err != nil { + return err + } + } +} + +// fetchOnce makes one GET and appends the body to tmp. It returns the bytes +// written. +func (r *runner) fetchOnce(ctx context.Context, j job, tmp string) (int64, error) { + f, err := os.OpenFile(tmp, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return 0, &writeError{err} + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return 0, &writeError{err} + } + offset := st.Size() + if j.size >= 0 && offset >= j.size { + if offset == j.size { + return 0, nil + } + offset = 0 + } + + attemptCtx, cancel := context.WithCancel(ctx) + defer cancel() + obj, err := r.client.GetObject(attemptCtx, j.url, offset) + if err != nil { + return 0, classifyStorageError(err, f) + } + defer obj.Body.Close() + if obj.Offset != offset { + offset = 0 + } + if j.size >= 0 && obj.Size >= 0 && obj.Size != j.size { + return 0, fmt.Errorf("storage has %d bytes, the export listed %d: %w", obj.Size, j.size, errNeedURL) + } + if err := f.Truncate(offset); err != nil { + return 0, &writeError{err} + } + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return 0, &writeError{err} + } + want := j.size + if want < 0 { + want = obj.Size + } + + stall := time.AfterFunc(stallTimeout, cancel) + defer stall.Stop() + var src io.Reader = &stallReader{r: obj.Body, timer: stall} + if want >= 0 { + // One byte more than expected shows an archive that grew. + src = io.LimitReader(src, want-offset+1) + } + w := &fileWriter{f: f} + n, err := io.Copy(w, src) + switch { + case w.err != nil: + return n, &writeError{w.err} + case err != nil && ctx.Err() == nil && attemptCtx.Err() != nil: + return n, errStalled + case err != nil: + return n, fmt.Errorf("%s: %w", api.OpDownloadArchive, err) + } + total := offset + n + switch { + case want >= 0 && total > want: + return n, fmt.Errorf("storage sent more than the listed %d bytes: %w", want, errNeedURL) + case want >= 0 && total < want: + return n, fmt.Errorf("%s: the archive ended after %d of %d bytes", api.OpDownloadArchive, total, want) + } + if err := f.Sync(); err != nil { + return n, &writeError{err} + } + return n, nil +} + +// classifyStorageError maps a failed GET: 403 means the URL expired, 416 that +// the partial file is longer than the archive (it is emptied for the retry). +func classifyStorageError(err error, f *os.File) error { + var se *api.StorageError + if !errors.As(err, &se) { + return err + } + switch se.StatusCode { + case http.StatusForbidden: + return fmt.Errorf("%v: %w", err, errNeedURL) + case http.StatusRequestedRangeNotSatisfiable: + if terr := f.Truncate(0); terr != nil { + return &writeError{terr} + } + } + return err +} + +// retryable reports whether another attempt may succeed. +func retryable(err error) bool { + var we *writeError + if errors.As(err, &we) || errors.Is(err, errNeedURL) { + return false + } + var se *api.StorageError + if errors.As(err, &se) && se.StatusCode != 0 { + return se.StatusCode >= 500 || se.StatusCode == http.StatusRequestedRangeNotSatisfiable || se.StatusCode == http.StatusTooManyRequests + } + return true +} + +// checkArchive checks the downloaded file's size and that it opens as a ZIP +// (its central directory reads), and returns the size. +func checkArchive(path string, want int64) (int64, error) { + st, err := os.Stat(path) + if err != nil { + return 0, &writeError{err} + } + if want >= 0 && st.Size() != want { + return 0, fmt.Errorf("the archive has %d bytes, the export listed %d", st.Size(), want) + } + zr, err := zip.OpenReader(path) + if err != nil { + return 0, fmt.Errorf("the downloaded archive is not a valid ZIP: %w", err) + } + zr.Close() + return st.Size(), nil +} + +func backoff(failures int) time.Duration { + d := time.Second << (failures - 1) + if d <= 0 || d > maxBackoff { + d = maxBackoff + } + return d +} + +// stallReader pushes back the stall timer whenever bytes arrive. +type stallReader struct { + r io.Reader + timer *time.Timer +} + +func (s *stallReader) Read(p []byte) (int, error) { + n, err := s.r.Read(p) + if n > 0 { + s.timer.Reset(stallTimeout) + } + return n, err +} + +// fileWriter remembers a write error, so it is not mistaken for a network one. +type fileWriter struct { + f *os.File + err error +} + +func (w *fileWriter) Write(p []byte) (int, error) { + n, err := w.f.Write(p) + if err != nil { + w.err = err + } + return n, err +} diff --git a/internal/export/export.go b/internal/export/export.go new file mode 100644 index 0000000..4c7b626 --- /dev/null +++ b/internal/export/export.go @@ -0,0 +1,467 @@ +// Package export follows an asset export on the dashboard and downloads its +// ZIP archives into a local folder, while the export still runs. A later run +// into the same folder skips the archives already there and resumes the ones +// left half-done. +package export + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/Codeinwp/optimole-cli/internal/api" +) + +// Concurrency limits: archives are up to about 1 GB each. +const ( + DefaultConcurrency = 2 + MaxConcurrency = 8 +) + +// maxURLRefreshes is how many fresh URLs one archive gets after storage +// refused one. +const maxURLRefreshes = 5 + +// Options configures a Run. +type Options struct { + Client *api.Client + ExportID int64 + // Dir is the folder the archives go into: one subfolder per domain. + Dir string + // Domains limits the download to these domains (case-insensitive); all + // when empty. + Domains []string + Concurrency int + // Sleep waits between looks at the export and between retries. Nil means + // a timer that stops early when the context is done. + Sleep func(ctx context.Context, d time.Duration) error + // OnEvent receives every event, from one goroutine. + OnEvent func(Event) +} + +// EventKind says what an Event reports. +type EventKind int + +// The events of a Run. +const ( + // ExportStatus: the export's status, when first seen and on each change. + ExportStatus EventKind = iota + // DomainStatus: a selected domain's status, when first seen and on each + // change, with the number of archives listed so far. + DomainStatus + // PartStarted: an archive download started. + PartStarted + // PartDone: an archive was downloaded, found already on disk, or failed. + PartDone +) + +// PartResult is how an archive ended. +type PartResult string + +// Archive results. +const ( + Downloaded PartResult = "downloaded" + Present PartResult = "present" + Failed PartResult = "failed" +) + +// Event is one step of a Run. +type Event struct { + Kind EventKind + // Status is the export's or the domain's status. + Status string + Domain string + // Parts is the number of archives a domain lists (DomainStatus). + Parts int + Part int64 + // Path is the archive's final path, under Dir. + Path string + // Size is the archive's size, or -1 when unknown. + Size int64 + Result PartResult + Err error + Elapsed time.Duration +} + +// Outcome is how a Run ended. +type Outcome struct { + // Export is the server's last answer, nil if none arrived. + Export *api.Export + // Folders maps each domain to its subfolder of Dir. + Folders map[string]string + // Counts of the selected archives the export listed. + Downloaded, Present, Failed int + // Bytes counts what this run wrote to disk. + Bytes int64 + // Err says why the run stopped before the export finished and every + // archive was tried: a failed look at the export, an *UnknownDomainError, + // the context, or the manifest that could not be written. + Err error +} + +// UnknownDomainError is a requested domain the export does not cover. +type UnknownDomainError struct { + Domain string + Known []string +} + +func (e *UnknownDomainError) Error() string { + return fmt.Sprintf("export has no domain %q (it has: %s)", e.Domain, strings.Join(e.Known, ", ")) +} + +type partState int + +const ( + pending partState = iota + active + done + needURL + failed +) + +type partKey struct { + domain string + number int64 +} + +// part is the local state of one listed archive. +type part struct { + key partKey + path string + // size and url are the latest the export listed (size -1: unknown). + size int64 + url string + state partState + result PartResult + doneSize int64 + refreshes int +} + +// finished is a download goroutine's report. +type finished struct { + part *part + res jobResult + elapsed time.Duration +} + +type runner struct { + client *api.Client + opts Options + sleep func(context.Context, time.Duration) error + exp *api.Export + folders folderMap + // wanted is the set of selected domains (nil: all). + wanted map[string]bool + domainStatus map[string]string + parts map[partKey]*part + queue []*part + bytes int64 +} + +// Run follows the export until it is final and every selected archive it +// lists is downloaded or failed, or until the context ends. Archives are +// downloaded as soon as the export lists them. An archive whose listed size +// changes after it was downloaded is downloaded again. +func Run(ctx context.Context, o Options) Outcome { + if o.Concurrency < 1 { + o.Concurrency = DefaultConcurrency + } + r := &runner{ + client: o.Client, + opts: o, + sleep: o.Sleep, + domainStatus: map[string]string{}, + parts: map[partKey]*part{}, + } + if r.sleep == nil { + r.sleep = sleepContext + } + out := r.loop(ctx) + out.Export = r.exp + out.Folders = r.folders.names + out.Bytes = r.bytes + for _, p := range r.parts { + switch { + case p.state == failed: + out.Failed++ + case p.state == done && p.result == Downloaded: + out.Downloaded++ + case p.state == done: + out.Present++ + } + } + var unknown *UnknownDomainError + if r.exp != nil && !errors.As(out.Err, &unknown) { + if err := writeManifest(o.Dir, r.exp, r.folders.names, r.parts); err != nil && out.Err == nil { + out.Err = err + } + } + return out +} + +// loop is the coordinator: it looks at the export, starts downloads and +// handles their results, all on one goroutine. +func (r *runner) loop(ctx context.Context) Outcome { + var out Outcome + started := time.Now() + results := make(chan finished) + pollDue := make(chan struct{}, 1) + pollNow, pollScheduled := true, false + running := 0 + defer func() { + // Wait for the downloads in flight; the context ends them early. + for ; running > 0; running-- { + r.finish(ctx, <-results) + } + }() + + for { + if pollNow { + pollNow = false + exp, err := r.client.GetExport(ctx, r.opts.ExportID) + if err == nil { + err = r.apply(exp) + } + if err != nil { + if ctx.Err() != nil { + err = ctx.Err() + } + out.Err = err + return out + } + } + for running < r.opts.Concurrency && len(r.queue) > 0 { + p := r.queue[0] + r.queue = r.queue[1:] + if p.state != pending { + continue + } + j := job{path: p.path, url: p.url, size: p.size} + if onDisk(j) { + r.finish(ctx, finished{part: p, res: jobResult{present: true, size: j.size}}) + continue + } + p.state = active + running++ + r.emit(Event{Kind: PartStarted, Domain: p.key.domain, Part: p.key.number, Path: p.path, Size: p.size}) + go func() { + start := time.Now() + res := r.download(ctx, j) + results <- finished{part: p, res: res, elapsed: time.Since(start)} + }() + } + final := api.IsFinalStatus(r.exp.Status) + refreshing := r.count(needURL) > 0 + if final && running == 0 && len(r.queue) == 0 && !refreshing { + return out + } + if (!final || refreshing) && !pollScheduled { + pollScheduled = true + wait := pollInterval(time.Since(started)) + if final { + wait = 0 + } + go func() { + r.sleep(ctx, wait) + pollDue <- struct{}{} + }() + } + select { + case f := <-results: + running-- + r.finish(ctx, f) + case <-pollDue: + pollScheduled, pollNow = false, true + case <-ctx.Done(): + out.Err = ctx.Err() + return out + } + } +} + +// apply takes in a fresh look at the export: status events, new archives to +// download, fresh URLs, and archives whose size changed. +func (r *runner) apply(exp *api.Export) error { + first := r.exp == nil + statusChanged := first || exp.Status != r.exp.Status + r.exp = exp + names := make([]string, 0, len(exp.Domains)) + for _, d := range exp.Domains { + names = append(names, d.Domain) + } + r.folders.add(names) + if first { + if err := r.selectDomains(names); err != nil { + return err + } + } + + listed := map[partKey]bool{} + for _, d := range exp.Domains { + if r.wanted != nil && !r.wanted[d.Domain] { + continue + } + parts := 0 + for _, lp := range d.Parts { + if !lp.Number.Valid || lp.Number.Value <= 0 || lp.URL == "" { + continue + } + parts++ + key := partKey{d.Domain, lp.Number.Value} + listed[key] = true + size := int64(-1) + if lp.Size.Valid && lp.Size.Value >= 0 { + size = lp.Size.Value + } + r.update(key, lp.URL, size) + } + if prev, seen := r.domainStatus[d.Domain]; !seen || prev != d.Status { + r.domainStatus[d.Domain] = d.Status + r.emit(Event{Kind: DomainStatus, Domain: d.Domain, Status: d.Status, Parts: parts}) + } + } + if api.IsFinalStatus(exp.Status) { + // A final export lists everything it will: an archive waiting for a + // URL the export no longer lists would wait forever. + for key, p := range r.parts { + if p.state == needURL && !listed[key] { + r.fail(p, errors.New("the export no longer lists this archive")) + } + } + } + // After the domains, so a final status comes last. + if statusChanged { + r.emit(Event{Kind: ExportStatus, Status: exp.Status}) + } + return nil +} + +// update records one listed archive and queues it when it needs downloading. +func (r *runner) update(key partKey, url string, size int64) { + p := r.parts[key] + if p == nil { + p = &part{ + key: key, + path: filepath.Join(r.opts.Dir, r.folders.names[key.domain], fmt.Sprintf("images-%d.zip", key.number)), + size: size, + url: url, + } + r.parts[key] = p + r.queue = append(r.queue, p) + return + } + p.url, p.size = url, size + switch { + case p.state == needURL: + p.state = pending + r.queue = append(r.queue, p) + case p.state == done && size >= 0 && size != p.doneSize: + p.state = pending + r.queue = append(r.queue, p) + } +} + +// selectDomains resolves the requested domains against the export's. +func (r *runner) selectDomains(names []string) error { + if len(r.opts.Domains) == 0 { + return nil + } + r.wanted = map[string]bool{} + for _, want := range r.opts.Domains { + found := false + for _, name := range names { + if strings.EqualFold(name, want) { + r.wanted[name] = true + found = true + } + } + if !found { + known := append([]string(nil), names...) + sort.Strings(known) + return &UnknownDomainError{Domain: want, Known: known} + } + } + return nil +} + +// finish handles a download's report. +func (r *runner) finish(ctx context.Context, f finished) { + p, res := f.part, f.res + r.bytes += res.written + switch { + case res.err == nil: + p.doneSize = res.size + p.result = Downloaded + if res.present { + p.result = Present + } + if p.size >= 0 && p.size != p.doneSize { + // The export listed a new size while this one downloaded. + p.state = pending + r.queue = append(r.queue, p) + return + } + p.state = done + r.emit(Event{Kind: PartDone, Domain: p.key.domain, Part: p.key.number, Path: p.path, Size: p.doneSize, Result: p.result, Elapsed: f.elapsed}) + case ctx.Err() != nil: + // Interrupted: neither done nor failed; the next run resumes it. + p.state = pending + case errors.Is(res.err, errNeedURL) && p.refreshes < maxURLRefreshes: + p.refreshes++ + p.state = needURL + default: + r.fail(p, res.err) + } +} + +func (r *runner) fail(p *part, err error) { + p.state = failed + p.result = Failed + r.emit(Event{Kind: PartDone, Domain: p.key.domain, Part: p.key.number, Path: p.path, Size: p.size, Result: Failed, Err: err}) +} + +func (r *runner) count(s partState) int { + n := 0 + for _, p := range r.parts { + if p.state == s { + n++ + } + } + return n +} + +func (r *runner) emit(e Event) { + if r.opts.OnEvent != nil { + r.opts.OnEvent(e) + } +} + +// pollInterval spaces the looks at a running export: every 5 seconds at +// first, like the dashboard, then less often for long exports. +func pollInterval(elapsed time.Duration) time.Duration { + switch { + case elapsed < 2*time.Minute: + return 5 * time.Second + case elapsed < 10*time.Minute: + return 15 * time.Second + } + return 30 * time.Second +} + +func sleepContext(ctx context.Context, d time.Duration) error { + if d <= 0 { + return ctx.Err() + } + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} diff --git a/internal/export/export_test.go b/internal/export/export_test.go new file mode 100644 index 0000000..a17b300 --- /dev/null +++ b/internal/export/export_test.go @@ -0,0 +1,624 @@ +package export + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Codeinwp/optimole-cli/internal/api" + "github.com/Codeinwp/optimole-cli/internal/testserver" +) + +// zipOf builds a ZIP archive holding the given files. +func zipOf(t *testing.T, files ...string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, name := range files { + w, err := zw.Create(name) + if err != nil { + t.Fatal(err) + } + fmt.Fprintf(w, "content of %s, padded so that ranges have room to land: %s", name, strings.Repeat("x", 200)) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +type fixture struct { + t *testing.T + srv *testserver.Server + client *api.Client + dir string + mu sync.Mutex + events []Event +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + srv := testserver.New(t) + c, err := api.New(api.Options{BaseURL: srv.URL, APIKey: testserver.DefaultKey, UserAgent: "optimole-cli/test", Sleep: immediate}) + if err != nil { + t.Fatal(err) + } + return &fixture{t: t, srv: srv, client: c, dir: filepath.Join(t.TempDir(), "optimole-export")} +} + +func immediate(ctx context.Context, _ time.Duration) error { return ctx.Err() } + +func (f *fixture) run(ctx context.Context, id int64, domains ...string) Outcome { + f.t.Helper() + f.mu.Lock() + f.events = nil + f.mu.Unlock() + return Run(ctx, Options{ + Client: f.client, + ExportID: id, + Dir: f.dir, + Domains: domains, + Concurrency: 2, + Sleep: immediate, + OnEvent: func(e Event) { + f.mu.Lock() + f.events = append(f.events, e) + f.mu.Unlock() + }, + }) +} + +func (f *fixture) file(rel string) []byte { + f.t.Helper() + b, err := os.ReadFile(filepath.Join(f.dir, filepath.FromSlash(rel))) + if err != nil { + f.t.Fatal(err) + } + return b +} + +func (f *fixture) downloads() []testserver.Event { + var out []testserver.Event + for _, e := range f.srv.Events() { + if e.Route == testserver.RouteDownload { + out = append(out, e) + } + } + return out +} + +func TestRunDownloadsArchivesWhileTheExportRuns(t *testing.T) { + f := newFixture(t) + a1, a2, d1 := zipOf(t, "example.com/a.jpg"), zipOf(t, "example.com/b.jpg"), zipOf(t, "c.png") + f.srv.SetExportDomains( + testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{a1, a2}}, + testserver.ExportDomain{Domain: "directUpload", Archives: [][]byte{d1}}, + ) + exp, err := f.client.CreateExport(context.Background()) + if err != nil { + t.Fatal(err) + } + out := f.run(context.Background(), exp.ID) + if out.Err != nil || out.Downloaded != 3 || out.Present != 0 || out.Failed != 0 { + t.Fatalf("outcome: %+v", out) + } + if out.Export.Status != api.ExportCompleted || out.Bytes != int64(len(a1)+len(a2)+len(d1)) { + t.Errorf("export %s, bytes %d", out.Export.Status, out.Bytes) + } + for rel, want := range map[string][]byte{"example.com/images-1.zip": a1, "example.com/images-2.zip": a2, "directUpload/images-1.zip": d1} { + if !bytes.Equal(f.file(rel), want) { + t.Errorf("%s differs", rel) + } + } + // The first archive started before the export finished. + started, completed := -1, -1 + for i, e := range f.events { + if e.Kind == PartStarted && started < 0 { + started = i + } + if e.Kind == ExportStatus && e.Status == api.ExportCompleted { + completed = i + } + } + if started < 0 || completed < 0 || started > completed { + t.Errorf("first download at event %d, export completed at %d", started, completed) + } + for i, e := range f.events { + if e.Kind == DomainStatus && i > completed { + t.Errorf("domain event %d (%s %s) after the export completed (%d)", i, e.Domain, e.Status, completed) + } + } + if leftovers, _ := filepath.Glob(filepath.Join(f.dir, "*", "*"+partSuffix)); len(leftovers) > 0 { + t.Errorf("partial files left: %v", leftovers) + } +} + +func TestRunSkipsArchivesAlreadyDownloaded(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{zipOf(t, "a"), zipOf(t, "b")}}) + if out := f.run(context.Background(), id); out.Downloaded != 2 { + t.Fatalf("first run: %+v", out) + } + before := len(f.downloads()) + out := f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != 0 || out.Present != 2 || out.Bytes != 0 { + t.Fatalf("second run: %+v", out) + } + if n := len(f.downloads()); n != before { + t.Errorf("second run downloaded again (%d requests)", n-before) + } + for _, e := range f.events { + if e.Kind == PartStarted { + t.Errorf("download started for an archive on disk: %+v", e) + } + } +} + +func TestRunResumesAPartialFile(t *testing.T) { + f := newFixture(t) + data := zipOf(t, "a.jpg", "b.jpg") + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + final := filepath.Join(f.dir, "example.com", "images-1.zip") + if err := os.MkdirAll(filepath.Dir(final), 0o755); err != nil { + t.Fatal(err) + } + tmp := tempPath(job{path: final, size: int64(len(data))}) + if err := os.WriteFile(tmp, data[:100], 0o644); err != nil { + t.Fatal(err) + } + // A partial file of another version of the archive is removed. + stale := final + ".12345" + partSuffix + os.WriteFile(stale, []byte("old"), 0o644) + + out := f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != 1 || out.Bytes != int64(len(data)-100) { + t.Fatalf("outcome: %+v", out) + } + if !bytes.Equal(f.file("example.com/images-1.zip"), data) { + t.Error("resumed archive differs") + } + if d := f.downloads(); len(d) != 1 || d[0].Range != "bytes=100-" { + t.Errorf("downloads: %+v", d) + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Errorf("stale partial file kept: %v", err) + } +} + +func TestRunRecoversFromStorageFailures(t *testing.T) { + data := zipOf(t, "a.jpg", "b.jpg") + cases := []struct { + name string + responses []testserver.Response + ranges []string + }{ + {"connection cut mid-body", []testserver.Response{{CutAfter: 50}}, []string{"", "bytes=50-"}}, + {"server error, then cut, then success", []testserver.Response{{Status: http.StatusInternalServerError}, {CutAfter: 30}}, []string{"", "", "bytes=30-"}}, + {"range ignored on resume", []testserver.Response{{CutAfter: 40}, {IgnoreRange: true}}, []string{"", "bytes=40-"}}, + {"expired URL", []testserver.Response{{Status: http.StatusForbidden, Body: "AccessDenied"}}, []string{"", ""}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + f.srv.Inject(testserver.RouteDownload, tc.responses...) + out := f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != 1 || out.Failed != 0 { + t.Fatalf("outcome: %+v (events %+v)", out, f.events) + } + if !bytes.Equal(f.file("example.com/images-1.zip"), data) { + t.Error("archive differs") + } + var ranges []string + for _, d := range f.downloads() { + ranges = append(ranges, d.Range) + } + if strings.Join(ranges, "|") != strings.Join(tc.ranges, "|") { + t.Errorf("ranges %q, want %q", ranges, tc.ranges) + } + }) + } +} + +func TestRunGetsAFreshURLWhenTheArchiveChanges(t *testing.T) { + old := zipOf(t, "a.jpg", "b.jpg") + for name, rewritten := range map[string][]byte{"bigger": zipOf(t, "a.jpg", "b.jpg", "c.jpg"), "smaller": zipOf(t, "a.jpg")} { + t.Run(name, func(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{old}}) + // The archive is rewritten between the listing and the download. + var once sync.Once + f.srv.Hook = func(route testserver.Route, _ *http.Request) { + if route == testserver.RouteDownload { + once.Do(func() { f.srv.RewriteArchive(id, "example.com", 1, rewritten) }) + } + } + out := f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != 1 { + t.Fatalf("outcome: %+v", out) + } + if !bytes.Equal(f.file("example.com/images-1.zip"), rewritten) { + t.Error("kept the old version") + } + if n, d := f.srv.Count(testserver.RouteExportGet), len(f.downloads()); n != 2 || d != 2 { + t.Errorf("looks at the export: %d, downloads %d, want 2 and 2", n, d) + } + }) + } +} + +func TestRunDownloadsAnArchiveAgainWhenItsSizeChanged(t *testing.T) { + f := newFixture(t) + a, b := zipOf(t, "a.jpg"), zipOf(t, "b.jpg") + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{a, b}}) + f.run(context.Background(), id) + bigger := zipOf(t, "b.jpg", "c.jpg") + f.srv.RewriteArchive(id, "example.com", 2, bigger) + out := f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != 1 || out.Present != 1 { + t.Fatalf("outcome: %+v", out) + } + if !bytes.Equal(f.file("example.com/images-2.zip"), bigger) { + t.Error("archive 2 not replaced") + } +} + +func TestRunGivesUpOnAnArchiveThatIsNotAZip(t *testing.T) { + f := newFixture(t) + good := zipOf(t, "a.jpg") + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{[]byte("not a zip at all"), good}}) + out := f.run(context.Background(), id) + if out.Downloaded != 1 || out.Failed != 1 || out.Err != nil { + t.Fatalf("outcome: %+v", out) + } + if _, err := os.Stat(filepath.Join(f.dir, "example.com", "images-1.zip")); !os.IsNotExist(err) { + t.Errorf("broken archive kept: %v", err) + } + if leftovers, _ := filepath.Glob(filepath.Join(f.dir, "example.com", "*"+partSuffix)); len(leftovers) > 0 { + t.Errorf("partial files left: %v", leftovers) + } + var failedErr error + for _, e := range f.events { + if e.Kind == PartDone && e.Result == Failed { + failedErr = e.Err + } + } + if failedErr == nil || !strings.Contains(failedErr.Error(), "not a valid ZIP") { + t.Errorf("failure: %v", failedErr) + } + tries := 0 + for _, d := range f.downloads() { + if strings.HasSuffix(d.Key, "/1") { + tries++ + } + } + if tries != 2 { + t.Errorf("downloads of the broken archive: %d, want 2", tries) + } +} + +func TestRunDoesNotRetryWhatCannotSucceed(t *testing.T) { + data := zipOf(t, "a.jpg") + t.Run("missing archive", func(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + f.srv.Inject(testserver.RouteDownload, testserver.Response{Status: http.StatusNotFound, Body: "NoSuchKey"}) + if out := f.run(context.Background(), id); out.Failed != 1 || len(f.downloads()) != 1 { + t.Fatalf("outcome %+v after %d downloads", out, len(f.downloads())) + } + }) + t.Run("unwritable folder", func(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + os.MkdirAll(f.dir, 0o755) + os.WriteFile(filepath.Join(f.dir, "example.com"), []byte("a file, not a folder"), 0o644) + out := f.run(context.Background(), id) + if out.Failed != 1 || len(f.downloads()) != 0 { + t.Fatalf("outcome %+v after %d downloads", out, len(f.downloads())) + } + }) + t.Run("URL refused every time", func(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + var denials []testserver.Response + for i := 0; i < 10; i++ { + denials = append(denials, testserver.Response{Status: http.StatusForbidden}) + } + f.srv.Inject(testserver.RouteDownload, denials...) + out := f.run(context.Background(), id) + if out.Failed != 1 || len(f.downloads()) != maxURLRefreshes+1 { + t.Fatalf("outcome %+v after %d downloads", out, len(f.downloads())) + } + }) +} + +func TestRunRestartsAStalledDownload(t *testing.T) { + saved := stallTimeout + stallTimeout = 100 * time.Millisecond + t.Cleanup(func() { stallTimeout = saved }) + f := newFixture(t) + data := zipOf(t, "a.jpg") + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + f.srv.Inject(testserver.RouteDownload, testserver.Response{StallAfter: 60}) + out := f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != 1 { + t.Fatalf("outcome: %+v", out) + } + if d := f.downloads(); len(d) != 2 || d[1].Range != "bytes=60-" { + t.Errorf("downloads: %+v", d) + } +} + +func TestRunKeepsTheOtherDomainsOfAFailedExport(t *testing.T) { + f := newFixture(t) + f.srv.SetExportDomains( + testserver.ExportDomain{Domain: "broken.com", Fail: true}, + testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{zipOf(t, "a.jpg")}}, + ) + exp, _ := f.client.CreateExport(context.Background()) + out := f.run(context.Background(), exp.ID) + if out.Err != nil || out.Export.Status != api.ExportFailed || out.Downloaded != 1 { + t.Fatalf("outcome: %+v", out) + } + var failedDomain bool + for _, e := range f.events { + if e.Kind == DomainStatus && e.Domain == "broken.com" && e.Status == api.ExportFailed { + failedDomain = true + } + } + if !failedDomain { + t.Errorf("no failed-domain event: %+v", f.events) + } +} + +func TestRunDomainFilter(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, + testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{zipOf(t, "a")}}, + testserver.ExportDomain{Domain: "other.com", Archives: [][]byte{zipOf(t, "b")}}, + ) + out := f.run(context.Background(), id, "EXAMPLE.com") + if out.Err != nil || out.Downloaded != 1 { + t.Fatalf("outcome: %+v", out) + } + if _, err := os.Stat(filepath.Join(f.dir, "other.com")); !os.IsNotExist(err) { + t.Errorf("other.com downloaded: %v", err) + } + + g := newFixture(t) + id = g.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{zipOf(t, "a")}}) + out = g.run(context.Background(), id, "nope.com") + var unknown *UnknownDomainError + if !errors.As(out.Err, &unknown) || unknown.Domain != "nope.com" || strings.Join(unknown.Known, ",") != "example.com" { + t.Fatalf("unknown domain: %v", out.Err) + } + if _, err := os.Stat(g.dir); !os.IsNotExist(err) { + t.Errorf("wrote into the folder for an unknown domain: %v", err) + } +} + +func TestRunKeepsHostileDomainNamesInsideTheFolder(t *testing.T) { + f := newFixture(t) + domains := []string{"../evil", "a/b", `c:\d`, "host:8080", "*.example.com", "CON", "con.example", "Example.com", "example.com", ".hidden", ".."} + var spec []testserver.ExportDomain + for _, d := range domains { + spec = append(spec, testserver.ExportDomain{Domain: d, Archives: [][]byte{zipOf(t, "x")}}) + } + id := f.srv.AddExport(api.ExportCompleted, spec...) + out := f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != len(domains) { + t.Fatalf("outcome: %+v", out) + } + seen := map[string]bool{} + for _, d := range domains { + name := out.Folders[d] + if name == "" || strings.ContainsAny(name, `/\:*?"<>|`) || strings.HasPrefix(name, ".") || seen[strings.ToLower(name)] { + t.Errorf("%q -> folder %q", d, name) + } + seen[strings.ToLower(name)] = true + } + entries, _ := os.ReadDir(f.dir) + if len(entries) != len(domains)+1 { // + export.json + t.Errorf("%d entries in the export folder, want %d", len(entries), len(domains)+1) + } + if _, err := os.Stat(filepath.Join(filepath.Dir(f.dir), "evil")); !os.IsNotExist(err) { + t.Errorf("wrote outside the export folder: %v", err) + } +} + +func TestFolderNames(t *testing.T) { + var m folderMap + m.add([]string{"example.com", "Example.com", "CON", "nul.txt", "*.x.com", "host:8080", "..", "", "bücher.de", strings.Repeat("a", 300)}) + want := map[string]string{ + "Example.com": "Example.com", + "example.com": "example.com-2", + "CON": "_CON", + "nul.txt": "_nul.txt", + "*.x.com": "_.x.com", + "host:8080": "host_8080", + "": "domain", + "..": "domain-2", + "bücher.de": "bücher.de", + } + for d, name := range want { + if m.names[d] != name { + t.Errorf("%q -> %q, want %q", d, m.names[d], name) + } + } + if got := m.names[strings.Repeat("a", 300)]; len(got) != maxFolderRunes { + t.Errorf("long name kept %d characters", len(got)) + } + // Domains seen later never take an earlier domain's folder. + m.add([]string{"EXAMPLE.COM"}) + if m.names["EXAMPLE.COM"] != "EXAMPLE.COM-3" || m.names["example.com"] != "example.com-2" { + t.Errorf("later domain: %v", m.names) + } +} + +func TestRunWritesAManifestWithoutURLs(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, + testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{zipOf(t, "a"), []byte("broken")}}, + testserver.ExportDomain{Domain: "skipped.com", Archives: [][]byte{zipOf(t, "b")}}, + ) + f.run(context.Background(), id, "example.com") + raw := f.file(ManifestName) + if bytes.Contains(raw, []byte("X-Amz-Signature")) || bytes.Contains(raw, []byte("http")) { + t.Errorf("manifest holds a URL: %s", raw) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.ID != id || m.Status != api.ExportCompleted || len(m.Domains) != 2 { + t.Fatalf("manifest: %+v", m) + } + ex := m.Domains[0] + if ex.Folder != "example.com" || len(ex.Archives) != 2 || !ex.Archives[0].Downloaded || ex.Archives[1].Downloaded || ex.Archives[0].File != "example.com/images-1.zip" { + t.Errorf("example.com: %+v", ex) + } + if sk := m.Domains[1]; len(sk.Archives) != 1 || sk.Archives[0].Downloaded { + t.Errorf("skipped.com: %+v", sk) + } +} + +func TestRunStopsWhenTheContextEnds(t *testing.T) { + f := newFixture(t) + f.srv.PauseExports(true) + id := f.srv.AddExport(api.ExportRunning, testserver.ExportDomain{Domain: "example.com"}) + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + out := f.run(ctx, id) + if !errors.Is(out.Err, context.DeadlineExceeded) || out.Export == nil || out.Export.Status != api.ExportRunning { + t.Fatalf("outcome: %+v", out) + } + if _, err := os.Stat(filepath.Join(f.dir, ManifestName)); err != nil { + t.Errorf("no manifest: %v", err) + } +} + +func TestRunReportsAMissingExport(t *testing.T) { + f := newFixture(t) + out := f.run(context.Background(), 404) + var apiErr *api.Error + if !errors.As(out.Err, &apiErr) || apiErr.StatusCode != http.StatusNotFound || out.Export != nil { + t.Fatalf("outcome: %+v", out) + } +} + +// newRunner returns a runner on the fixture, for tests of single steps. +func (f *fixture) newRunner() *runner { + return &runner{client: f.client, opts: Options{Dir: f.dir, Concurrency: 1}, sleep: immediate, domainStatus: map[string]string{}, parts: map[partKey]*part{}} +} + +func TestDownloadRefusesAnArchiveLongerThanListed(t *testing.T) { + data := zipOf(t, "a.jpg") + storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // No Content-Length: the body is chunked. + w.WriteHeader(http.StatusOK) + w.Write(data) + w.(http.Flusher).Flush() + w.Write([]byte("more than the export listed")) + })) + defer storage.Close() + f := newFixture(t) + path := filepath.Join(f.dir, "example.com", "images-1.zip") + res := f.newRunner().download(context.Background(), job{path: path, url: storage.URL + "/x", size: int64(len(data))}) + if !errors.Is(res.err, errNeedURL) { + t.Fatalf("got %+v", res) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("archive put in place: %v", err) + } +} + +func TestDownloadEmptiesAPartialFileLongerThanTheArchive(t *testing.T) { + f := newFixture(t) + data := zipOf(t, "a.jpg") + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + exp, err := f.client.GetExport(context.Background(), id) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(f.dir, "example.com", "images-1.zip") + j := job{path: path, url: exp.Domains[0].Parts[0].URL, size: -1} + os.MkdirAll(filepath.Dir(path), 0o755) + os.WriteFile(tempPath(j), bytes.Repeat([]byte("x"), len(data)+10), 0o644) + res := f.newRunner().download(context.Background(), j) + if res.err != nil || res.size != int64(len(data)) || !bytes.Equal(f.file("example.com/images-1.zip"), data) { + t.Fatalf("got %+v", res) + } + if d := f.downloads(); len(d) != 2 || d[0].Range == "" || d[1].Range != "" { + t.Errorf("downloads: %+v", d) + } +} + +func TestFinishQueuesAnArchiveListedWithANewSizeDuringItsDownload(t *testing.T) { + f := newFixture(t) + r := f.newRunner() + p := &part{key: partKey{"example.com", 1}, size: 200, state: active} + r.finish(context.Background(), finished{part: p, res: jobResult{size: 100}}) + if p.state != pending || len(r.queue) != 1 { + t.Errorf("state %v, queue %d", p.state, len(r.queue)) + } +} + +func TestApplyFailsAWaitingArchiveTheFinalExportNoLongerLists(t *testing.T) { + f := newFixture(t) + r := f.newRunner() + r.exp = &api.Export{ID: 1, Status: api.ExportRunning} + waiting := &part{key: partKey{"example.com", 2}, state: needURL} + r.parts[waiting.key] = waiting + if err := r.apply(&api.Export{ID: 1, Status: api.ExportCompleted, Domains: []api.ExportDomain{{Domain: "example.com", Status: api.ExportCompleted}}}); err != nil { + t.Fatal(err) + } + if waiting.state != failed { + t.Errorf("state %v", waiting.state) + } +} + +func TestUpdateQueuesADownloadedArchiveListedWithANewSize(t *testing.T) { + f := newFixture(t) + r := f.newRunner() + key := partKey{"example.com", 1} + p := &part{key: key, size: 100, doneSize: 100, state: done} + r.parts[key] = p + r.update(key, "https://storage/new", 100) + if p.state != done || len(r.queue) != 0 { + t.Fatalf("same size: state %v, queue %d", p.state, len(r.queue)) + } + r.update(key, "https://storage/new", 200) + if p.state != pending || len(r.queue) != 1 || p.url != "https://storage/new" { + t.Errorf("new size: state %v, queue %d", p.state, len(r.queue)) + } +} + +func TestFetchNamesAStalledDownload(t *testing.T) { + saved := stallTimeout + stallTimeout = 50 * time.Millisecond + t.Cleanup(func() { stallTimeout = saved }) + storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "100") + w.Write([]byte("0123456789")) + w.(http.Flusher).Flush() + <-r.Context().Done() + })) + defer storage.Close() + f := newFixture(t) + os.MkdirAll(f.dir, 0o755) + n, err := f.newRunner().fetchOnce(context.Background(), job{path: filepath.Join(f.dir, "a.zip"), url: storage.URL + "/x", size: 100}, filepath.Join(f.dir, "a.zip.part")) + if !errors.Is(err, errStalled) || n != 10 { + t.Fatalf("got %d bytes, %v", n, err) + } +} diff --git a/internal/export/manifest.go b/internal/export/manifest.go new file mode 100644 index 0000000..8fcc42f --- /dev/null +++ b/internal/export/manifest.go @@ -0,0 +1,84 @@ +package export + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/Codeinwp/optimole-cli/internal/api" +) + +// ManifestName is the file next to the domain folders that describes the +// export: its status and, for each domain, its folder and archives. It holds +// no download URLs. +const ManifestName = "export.json" + +type manifest struct { + ID int64 `json:"id"` + Status string `json:"status"` + CreatedAt string `json:"created_at,omitempty"` + Domains []manifestDomain `json:"domains"` +} + +type manifestDomain struct { + Domain string `json:"domain"` + Folder string `json:"folder"` + Status string `json:"status"` + Archives []manifestArchive `json:"archives"` +} + +type manifestArchive struct { + PartNumber int64 `json:"part_number"` + // File is relative to the export folder, "/"-separated. + File string `json:"file"` + // Size is the size the export listed, -1 when it gave none. + Size int64 `json:"size"` + // Downloaded: the file on disk is this archive, complete and checked. + Downloaded bool `json:"downloaded"` +} + +// writeManifest writes ManifestName into dir from the export's last answer, +// replacing an earlier one in one step. +func writeManifest(dir string, exp *api.Export, folders map[string]string, parts map[partKey]*part) error { + m := manifest{ID: exp.ID, Status: exp.Status, CreatedAt: exp.CreatedAt, Domains: []manifestDomain{}} + for _, d := range exp.Domains { + md := manifestDomain{Domain: d.Domain, Folder: folders[d.Domain], Status: d.Status, Archives: []manifestArchive{}} + for _, lp := range d.Parts { + if !lp.Number.Valid || lp.Number.Value <= 0 { + continue + } + ma := manifestArchive{ + PartNumber: lp.Number.Value, + File: md.Folder + "/" + fmt.Sprintf("images-%d.zip", lp.Number.Value), + Size: -1, + } + if lp.Size.Valid { + ma.Size = lp.Size.Value + } + if p := parts[partKey{d.Domain, lp.Number.Value}]; p != nil && p.state == done && (ma.Size < 0 || ma.Size == p.doneSize) { + ma.Downloaded = true + } + md.Archives = append(md.Archives, ma) + } + m.Domains = append(m.Domains, md) + } + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("writing %s: %w", ManifestName, err) + } + path := filepath.Join(dir, ManifestName) + tmp := path + partSuffix + if err := os.WriteFile(tmp, append(data, '\n'), 0o644); err != nil { + os.Remove(tmp) + return fmt.Errorf("writing %s: %w", ManifestName, err) + } + if err := os.Rename(tmp, path); err != nil { + os.Remove(tmp) + return fmt.Errorf("writing %s: %w", ManifestName, err) + } + return nil +} diff --git a/internal/export/names.go b/internal/export/names.go new file mode 100644 index 0000000..8824ea7 --- /dev/null +++ b/internal/export/names.go @@ -0,0 +1,79 @@ +package export + +import ( + "fmt" + "sort" + "strings" + "unicode" +) + +// maxFolderRunes keeps folder names well inside path length limits. +const maxFolderRunes = 100 + +// windowsReserved are device names Windows refuses as a file name, with or +// without an extension. +var windowsReserved = map[string]bool{ + "CON": true, "PRN": true, "AUX": true, "NUL": true, + "COM1": true, "COM2": true, "COM3": true, "COM4": true, "COM5": true, "COM6": true, "COM7": true, "COM8": true, "COM9": true, + "LPT1": true, "LPT2": true, "LPT3": true, "LPT4": true, "LPT5": true, "LPT6": true, "LPT7": true, "LPT8": true, "LPT9": true, +} + +// folderMap maps each domain of an export to the local folder its archives +// go into. Domain names come from the server, so each becomes one safe path +// segment (see safeSegment). Names that clash case-insensitively, as they do +// on macOS and Windows, get "-2", "-3"... in sorted domain order, so the same +// export maps the same way on every run. +type folderMap struct { + names map[string]string + taken map[string]bool +} + +// add maps the domains not mapped yet. +func (m *folderMap) add(domains []string) { + if m.names == nil { + m.names, m.taken = map[string]string{}, map[string]bool{} + } + sorted := append([]string(nil), domains...) + sort.Strings(sorted) + for _, d := range sorted { + if _, done := m.names[d]; done { + continue + } + base := safeSegment(d) + name := base + for n := 2; m.taken[strings.ToLower(name)]; n++ { + name = fmt.Sprintf("%s-%d", base, n) + } + m.taken[strings.ToLower(name)] = true + m.names[d] = name + } +} + +// safeSegment turns a domain name into a file name that is valid and inert on +// every platform: letters, digits, ".", "-" and "_" only (anything else +// becomes "_"), at most maxFolderRunes long, no leading or trailing dot, and +// no Windows device name. +func safeSegment(domain string) string { + var b strings.Builder + n := 0 + for _, r := range domain { + if n == maxFolderRunes { + break + } + if r == '.' || r == '-' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + } else { + b.WriteRune('_') + } + n++ + } + name := strings.Trim(b.String(), ".") + if name == "" { + return "domain" + } + stem, _, _ := strings.Cut(name, ".") + if windowsReserved[strings.ToUpper(stem)] { + name = "_" + name + } + return name +} diff --git a/internal/output/export.go b/internal/output/export.go new file mode 100644 index 0000000..f219079 --- /dev/null +++ b/internal/output/export.go @@ -0,0 +1,209 @@ +package output + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/Codeinwp/optimole-cli/internal/api" + "github.com/Codeinwp/optimole-cli/internal/export" + "github.com/Codeinwp/optimole-cli/internal/upload" +) + +// ExportReporter prints the events of an export download as lines or JSON +// lines. It is not safe for concurrent use; export.Run reports from one +// goroutine. +type ExportReporter struct { + w *ErrWriter + json bool + enc *json.Encoder + redact func(string) string + id int64 +} + +// NewExportReporter returns a reporter for export id writing to w. redact is +// applied to every error message (nil means none). +func NewExportReporter(w io.Writer, jsonLines bool, id int64, redact func(string) string) *ExportReporter { + if redact == nil { + redact = func(s string) string { return s } + } + ew := NewErrWriter(w) + enc := json.NewEncoder(ew) + enc.SetEscapeHTML(false) + return &ExportReporter{w: ew, json: jsonLines, enc: enc, redact: redact, id: id} +} + +// Err returns the first error writing the output, if any. +func (r *ExportReporter) Err() error { return r.w.Err() } + +// ExportLine is the JSON object printed for each event. +type ExportLine struct { + Event string `json:"event"` + ID int64 `json:"id,omitempty"` + Created *bool `json:"created,omitempty"` + Domain string `json:"domain,omitempty"` + Part int64 `json:"part,omitempty"` + Path string `json:"path,omitempty"` + Status string `json:"status,omitempty"` + Archives *int `json:"archives,omitempty"` + Size *int64 `json:"size,omitempty"` + Seconds *int64 `json:"seconds,omitempty"` + Error string `json:"error,omitempty"` + Code string `json:"code,omitempty"` +} + +// ExportSummary is the final JSON object. +type ExportSummary struct { + ID int64 `json:"id"` + Status string `json:"status"` + Dir string `json:"dir"` + Downloaded int `json:"downloaded"` + Present int `json:"present"` + Failed int `json:"failed"` + Bytes int64 `json:"bytes"` + FailedDomains []string `json:"failed_domains"` + Interrupted bool `json:"interrupted"` + TimedOut bool `json:"timed_out"` +} + +// Started reports the export the command follows: created by it, or the +// one already running. +func (r *ExportReporter) Started(created bool) { + if r.json { + r.enc.Encode(ExportLine{Event: "started", ID: r.id, Created: &created}) + return + } + if created { + fmt.Fprintf(r.w, "Started export %d.\n", r.id) + } else { + fmt.Fprintf(r.w, "Export %d is already running; following it.\n", r.id) + } +} + +// Event reports one step of export.Run. +func (r *ExportReporter) Event(e export.Event) { + if r.json { + r.enc.Encode(r.line(e)) + return + } + switch e.Kind { + case export.ExportStatus: + fmt.Fprintf(r.w, "Export %d: %s\n", r.id, e.Status) + case export.DomainStatus: + fmt.Fprintf(r.w, " %s: %s\n", e.Domain, domainDetail(e)) + case export.PartStarted: + fmt.Fprintf(r.w, "downloading %s%s\n", e.Path, sizeNote(e.Size, "")) + case export.PartDone: + switch e.Result { + case export.Downloaded: + fmt.Fprintf(r.w, "downloaded %s%s\n", e.Path, sizeNote(e.Size, " in "+e.Elapsed.Round(time.Second).String())) + case export.Present: + fmt.Fprintf(r.w, "skipped %s: already downloaded\n", e.Path) + default: + fmt.Fprintf(r.w, "failed %s: %s\n", e.Path, r.errText(e.Err)) + } + } +} + +func (r *ExportReporter) line(e export.Event) ExportLine { + switch e.Kind { + case export.ExportStatus: + return ExportLine{Event: "export", ID: r.id, Status: e.Status} + case export.DomainStatus: + n := e.Parts + return ExportLine{Event: "domain", Domain: e.Domain, Status: e.Status, Archives: &n} + } + l := ExportLine{Event: "archive", Domain: e.Domain, Part: e.Part, Path: e.Path, Status: string(e.Result)} + if e.Kind == export.PartStarted { + l.Event, l.Status = "downloading", "" + } + if e.Size >= 0 { + size := e.Size + l.Size = &size + } + if e.Result == export.Downloaded { + secs := int64(e.Elapsed.Round(time.Second) / time.Second) + l.Seconds = &secs + } + if e.Err != nil { + l.Error = r.errText(e.Err) + var se *api.StorageError + if errors.As(e.Err, &se) { + l.Code = se.Code + } + } + return l +} + +// Finish prints the summary of a run into dir. +func (r *ExportReporter) Finish(out export.Outcome, dir string, interrupted, timedOut bool) { + s := ExportSummary{ + ID: r.id, Dir: dir, Downloaded: out.Downloaded, Present: out.Present, Failed: out.Failed, + Bytes: out.Bytes, FailedDomains: []string{}, Interrupted: interrupted, TimedOut: timedOut, + } + if out.Export != nil { + s.Status = out.Export.Status + for _, d := range out.Export.Domains { + if d.Status == api.ExportFailed || d.Status == api.ExportCancelled { + s.FailedDomains = append(s.FailedDomains, d.Domain) + } + } + } + if r.json { + r.enc.Encode(struct { + Summary ExportSummary `json:"summary"` + }{s}) + return + } + var b strings.Builder + fmt.Fprintf(&b, "\nExport %d", r.id) + if s.Status != "" { + fmt.Fprintf(&b, " %s", s.Status) + } + if len(s.FailedDomains) > 0 { + fmt.Fprintf(&b, " (failed: %s)", strings.Join(s.FailedDomains, ", ")) + } + fmt.Fprintf(&b, ": %d downloaded, %d already there, %d failed (%s written) in %s.", s.Downloaded, s.Present, s.Failed, upload.HumanSize(s.Bytes), dir) + if interrupted { + b.WriteString(" Interrupted.") + } + if timedOut { + b.WriteString(" Timed out.") + } + fmt.Fprintln(r.w, b.String()) +} + +func (r *ExportReporter) errText(err error) string { + if err == nil { + return "" + } + return r.redact(err.Error()) +} + +func domainDetail(e export.Event) string { + if !api.IsFinalStatus(e.Status) { + return e.Status + } + switch { + case e.Parts == 0 && e.Status == api.ExportCompleted: + return e.Status + ", no files" + case e.Parts == 0: + return e.Status + case e.Parts == 1: + return e.Status + ", 1 archive" + } + return fmt.Sprintf("%s, %d archives", e.Status, e.Parts) +} + +func sizeNote(size int64, extra string) string { + if size < 0 { + if extra == "" { + return "" + } + return " (" + strings.TrimPrefix(extra, " ") + ")" + } + return " (" + upload.HumanSize(size) + extra + ")" +} diff --git a/internal/output/output.go b/internal/output/output.go index 4991d6f..7646083 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -1,5 +1,6 @@ -// Package output prints upload results as human-readable lines or as JSON -// lines, and keeps the counts for the final summary. +// Package output prints the results of uploads and export downloads as +// human-readable lines or as JSON lines, and keeps the counts for the final +// summary. package output import ( diff --git a/internal/testserver/exports.go b/internal/testserver/exports.go new file mode 100644 index 0000000..a3601b5 --- /dev/null +++ b/internal/testserver/exports.go @@ -0,0 +1,474 @@ +package testserver + +import ( + "fmt" + "net/http" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// The fake's export endpoints. +const ( + RouteExportCreate Route = "export-create" + RouteExportGet Route = "export-get" + RouteExportList Route = "export-list" + RouteExportDelete Route = "export-delete" + RouteDownload Route = "download" +) + +// ExportLimit is how many exports an account keeps. +const ExportLimit = 5 + +// ExportDomain is a domain of a fake export and the ZIP archives it ends with. +type ExportDomain struct { + Domain string + Archives [][]byte + // Fail ends the domain as failed once its archives are listed. + Fail bool +} + +// fakeExport is an export the fake runs. Every look at it (GET) moves it on: +// queued -> running, then one more archive per domain, then each domain +// ends, and the export's status follows the dashboard's rules. +type fakeExport struct { + id int64 + status string + created time.Time + domains []*fakeExportDomain +} + +type fakeExportDomain struct { + name string + archives [][]byte + listed int + fail bool + status string +} + +// SetExportDomains sets the domains the next created exports cover (default: +// "directUpload" with no archives). +func (s *Server) SetExportDomains(domains ...ExportDomain) { + s.mu.Lock() + defer s.mu.Unlock() + s.exportTemplate = domains +} + +// AddExport stores an export in the given status, its archives all listed +// when the status is final, and returns its ID. +func (s *Server) AddExport(status string, domains ...ExportDomain) int64 { + s.mu.Lock() + defer s.mu.Unlock() + e := s.newExportLocked(domains) + e.status = status + if finalStatus(status) { + for _, d := range e.domains { + d.listed = len(d.archives) + d.status = "completed" + if d.fail { + d.status = "failed" + } + } + } + return e.id +} + +// PauseExports stops (or restarts) the progress of running exports. +func (s *Server) PauseExports(paused bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.exportsPaused = paused +} + +// RewriteArchive replaces the bytes of a listed archive, as a repeated server +// step does. +func (s *Server) RewriteArchive(id int64, domain string, part int, data []byte) { + s.mu.Lock() + defer s.mu.Unlock() + for _, d := range s.exports[id].domains { + if d.name == domain { + d.archives[part-1] = data + } + } +} + +// ExpireArchiveURLs makes every archive URL handed out so far fail with 403. +func (s *Server) ExpireArchiveURLs() { + s.mu.Lock() + defer s.mu.Unlock() + s.urlGeneration++ +} + +// ExportIDs lists the stored exports, oldest first. +func (s *Server) ExportIDs() []int64 { + s.mu.Lock() + defer s.mu.Unlock() + ids := make([]int64, 0, len(s.exports)) + for id := range s.exports { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + return ids +} + +func finalStatus(status string) bool { + return status == "completed" || status == "failed" || status == "cancelled" +} + +func (s *Server) newExportLocked(domains []ExportDomain) *fakeExport { + if s.exports == nil { + s.exports = map[int64]*fakeExport{} + } + s.nextExport++ + e := &fakeExport{ + id: s.nextExport, + status: "queued", + created: time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC).Add(time.Duration(s.nextExport) * time.Minute), + } + if len(domains) == 0 { + domains = []ExportDomain{{Domain: "directUpload"}} + } + for _, d := range domains { + e.domains = append(e.domains, &fakeExportDomain{name: d.Domain, archives: d.Archives, fail: d.Fail, status: "queued"}) + } + s.exports[e.id] = e + return e +} + +// advanceLocked moves a running export one step on. +func (e *fakeExport) advanceLocked() { + if finalStatus(e.status) { + return + } + if e.status == "queued" { + e.status = "running" + for _, d := range e.domains { + d.status = "running" + } + return + } + counts := map[string]int{} + for _, d := range e.domains { + if d.status == "running" { + if d.listed < len(d.archives) { + d.listed++ + } else if d.fail { + d.status = "failed" + } else { + d.status = "completed" + } + } + counts[d.status]++ + } + switch { + case counts["running"] > 0: + e.status = "running" + case counts["queued"] > 0: + e.status = "queued" + case counts["failed"] > 0: + e.status = "failed" + case counts["cancelled"] > 0: + e.status = "cancelled" + default: + e.status = "completed" + } +} + +func (s *Server) exportSummaryLocked(e *fakeExport) map[string]any { + names := make([]string, 0, len(e.domains)) + for _, d := range e.domains { + names = append(names, d.name) + } + return map[string]any{"id": e.id, "status": e.status, "created_at": e.created.Format("2006-01-02T15:04:05.000000Z"), "domains": names} +} + +func (s *Server) exportDetailLocked(e *fakeExport) map[string]any { + domains := []map[string]any{} + for _, d := range e.domains { + files := []map[string]any{} + for i := 0; i < d.listed; i++ { + files = append(files, map[string]any{ + "part_number": i + 1, + "url": s.archiveURLLocked(e.id, d.name, i+1), + "size": len(d.archives[i]), + }) + } + domains = append(domains, map[string]any{"domain": d.name, "status": d.status, "zip_files": files}) + } + out := s.exportSummaryLocked(e) + out["domains"] = domains + return out +} + +func (s *Server) archiveURLLocked(id int64, domain string, part int) string { + return fmt.Sprintf("%s/exports/%d/%s/images-%d.zip?X-Amz-Signature=%s", s.S3.URL, id, url.PathEscape(domain), part, s.archiveSigLocked(id, domain, part)) +} + +func (s *Server) archiveSigLocked(id int64, domain string, part int) string { + return fmt.Sprintf("sig-%d-%d-%x-%d", s.urlGeneration, id, domain, part) +} + +// exportAuth authenticates an export request: full-access keys only. +func (s *Server) exportAuth(w http.ResponseWriter, r *http.Request) (int, bool) { + key, ok := s.authenticate(w, r) + if !ok { + return http.StatusUnauthorized, false + } + if key.Scoped { + return writeJSON(w, http.StatusForbidden, map[string]string{"message": "Full API access required"}), false + } + return 0, true +} + +// handleExports serves GET (list) and POST (create) /api/asset-exports. +func (s *Server) handleExports(w http.ResponseWriter, r *http.Request) { + route := RouteExportList + if r.Method == http.MethodPost { + route = RouteExportCreate + } + ev, injected := s.begin(route, r) + status := 0 + var drop func() + defer func() { s.finishThenDrop(ev, status, drop) }() + if injected != nil && injected.CommitThenClose { + w, drop = commitThenClose(w) + injected = nil + } + if injected != nil { + status, drop = writeInjected(w, injected) + return + } + var ok bool + if status, ok = s.exportAuth(w, r); !ok { + return + } + switch r.Method { + case http.MethodPost: + status = s.createExport(w) + case http.MethodGet: + status = s.listExports(w, r) + default: + status = writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"message": "Method not allowed."}) + } +} + +func (s *Server) createExport(w http.ResponseWriter) int { + s.mu.Lock() + defer s.mu.Unlock() + var latest *fakeExport + for _, e := range s.exports { + if latest == nil || e.id > latest.id { + latest = e + } + } + legacy := s.LegacyExportConflicts + if latest != nil && !finalStatus(latest.status) { + body := map[string]any{"message": "Asset export is already in progress"} + if !legacy { + body["code"] = "export_in_progress" + body["export_id"] = latest.id + } + return writeJSON(w, http.StatusConflict, body) + } + if len(s.exports) >= ExportLimit { + body := map[string]any{"message": fmt.Sprintf("Maximum asset export count reached (Limit: %d)", ExportLimit)} + if !legacy { + body["code"] = "export_limit_reached" + body["limit"] = ExportLimit + body["count"] = len(s.exports) + } + return writeJSON(w, http.StatusConflict, body) + } + e := s.newExportLocked(s.exportTemplate) + return writeJSON(w, http.StatusCreated, s.exportSummaryLocked(e)) +} + +func (s *Server) listExports(w http.ResponseWriter, r *http.Request) int { + q := r.URL.Query() + perPage := 20 + if v := q.Get("per_page"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > 100 { + return validation(w, "per_page", "The per page field must be between 1 and 100.") + } + perPage = n + } + var before int64 = 1 << 62 + if c := q.Get("cursor"); c != "" { + n, err := strconv.ParseInt(c, 10, 64) + if err != nil { + return validation(w, "cursor", "The cursor is invalid.") + } + before = n + } + s.mu.Lock() + defer s.mu.Unlock() + ids := make([]int64, 0, len(s.exports)) + for id := range s.exports { + if id < before { + ids = append(ids, id) + } + } + sort.Slice(ids, func(i, j int) bool { return ids[i] > ids[j] }) + data := []map[string]any{} + for _, id := range ids { + if len(data) == perPage { + break + } + data = append(data, s.exportSummaryLocked(s.exports[id])) + } + more := len(ids) > len(data) + var next any + if more { + next = strconv.FormatInt(data[len(data)-1]["id"].(int64), 10) + } + return writeJSON(w, http.StatusOK, map[string]any{ + "data": data, + "meta": map[string]any{"per_page": perPage, "prev_cursor": nil, "next_cursor": next, "has_more": more}, + }) +} + +var exportPathPattern = regexp.MustCompile(`^/api/asset-exports/([0-9]+)$`) + +// handleExport serves GET and DELETE /api/asset-exports/{id}. +func (s *Server) handleExport(w http.ResponseWriter, r *http.Request) { + route := RouteExportGet + if r.Method == http.MethodDelete { + route = RouteExportDelete + } + ev, injected := s.begin(route, r) + status := 0 + var drop func() + defer func() { s.finishThenDrop(ev, status, drop) }() + if injected != nil && injected.CommitThenClose { + w, drop = commitThenClose(w) + injected = nil + } + if injected != nil { + status, drop = writeInjected(w, injected) + return + } + var ok bool + if status, ok = s.exportAuth(w, r); !ok { + return + } + m := exportPathPattern.FindStringSubmatch(r.URL.Path) + if m == nil { + status = writeJSON(w, http.StatusNotFound, map[string]string{"message": "Not found."}) + return + } + id, _ := strconv.ParseInt(m[1], 10, 64) + s.mu.Lock() + defer s.mu.Unlock() + e := s.exports[id] + if e == nil { + status = writeJSON(w, http.StatusNotFound, map[string]string{"message": "Not found."}) + return + } + switch r.Method { + case http.MethodGet: + if !s.exportsPaused { + e.advanceLocked() + } + status = writeJSON(w, http.StatusOK, s.exportDetailLocked(e)) + case http.MethodDelete: + if !finalStatus(e.status) { + status = writeJSON(w, http.StatusConflict, map[string]string{"message": fmt.Sprintf("Asset export with status %q cannot be deleted", e.status)}) + return + } + delete(s.exports, id) + w.WriteHeader(http.StatusNoContent) + status = http.StatusNoContent + default: + status = writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"message": "Method not allowed."}) + } +} + +var archivePathPattern = regexp.MustCompile(`^/exports/([0-9]+)/([^/]+)/images-([0-9]+)\.zip$`) + +// handleDownload serves archive GETs like S3: the signature must be current, +// "Range: bytes=N-" gets a 206, a range past the end a 416. +func (s *Server) handleDownload(w http.ResponseWriter, r *http.Request) { + ev, injected := s.begin(RouteDownload, r) + ev.Range = r.Header.Get("Range") + status := 0 + var drop func() + defer func() { s.finishThenDrop(ev, status, drop) }() + if r.Header.Get("Authorization") != "" { + s.violate("archive download carried an Authorization header") + } + if injected != nil && injected.CutAfter == 0 && !injected.IgnoreRange && injected.StallAfter == 0 { + status, drop = writeInjected(w, injected) + return + } + s3Error := func(code int, name string) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(code) + fmt.Fprintf(w, "\n%s%s", name, name) + status = code + } + m := archivePathPattern.FindStringSubmatch(r.URL.EscapedPath()) + if m == nil || r.Method != http.MethodGet { + s3Error(http.StatusForbidden, "AccessDenied") + return + } + id, _ := strconv.ParseInt(m[1], 10, 64) + domain, _ := url.PathUnescape(m[2]) + part, _ := strconv.Atoi(m[3]) + ev.Key = fmt.Sprintf("%d/%s/%d", id, domain, part) + s.mu.Lock() + var data []byte + if e := s.exports[id]; e != nil && r.URL.Query().Get("X-Amz-Signature") == s.archiveSigLocked(id, domain, part) { + for _, d := range e.domains { + if d.name == domain && part >= 1 && part <= d.listed { + data = d.archives[part-1] + } + } + } + s.mu.Unlock() + if data == nil { + s3Error(http.StatusForbidden, "AccessDenied") + return + } + + start := int64(0) + if rg := r.Header.Get("Range"); rg != "" && (injected == nil || !injected.IgnoreRange) { + n, err := strconv.ParseInt(strings.TrimSuffix(strings.TrimPrefix(rg, "bytes="), "-"), 10, 64) + if err != nil || !strings.HasPrefix(rg, "bytes=") || !strings.HasSuffix(rg, "-") { + s.violate("unexpected Range header %q", rg) + } + if n >= int64(len(data)) { + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(data))) + s3Error(http.StatusRequestedRangeNotSatisfiable, "InvalidRange") + return + } + start = n + } + body := data[start:] + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + if start > 0 { + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + status = http.StatusPartialContent + } else { + w.WriteHeader(http.StatusOK) + status = http.StatusOK + } + switch { + case injected != nil && injected.CutAfter > 0 && injected.CutAfter < int64(len(body)): + w.Write(body[:injected.CutAfter]) + // Flushed first: hijacking discards what is still buffered. + w.(http.Flusher).Flush() + drop = func() { dropConn(w) } + case injected != nil && injected.StallAfter > 0 && injected.StallAfter < int64(len(body)): + w.Write(body[:injected.StallAfter]) + w.(http.Flusher).Flush() + <-r.Context().Done() + default: + w.Write(body) + } +} diff --git a/internal/testserver/testserver.go b/internal/testserver/testserver.go index f24ed96..5f0d2d7 100644 --- a/internal/testserver/testserver.go +++ b/internal/testserver/testserver.go @@ -1,7 +1,8 @@ // Package testserver is an in-memory fake of the dashboard's unified upload API // and of the presigned storage endpoint, for tests only. It follows the API // contract: POST /api/assets, PUT to the presigned URL, POST -// /api/assets/{key}/completion and GET /api/folders. +// /api/assets/{key}/completion, GET /api/folders, the /api/asset-exports +// endpoints and GETs of the export archives. package testserver import ( @@ -51,6 +52,15 @@ type Response struct { // storing the bytes) and then drops the connection instead of answering: // the client cannot tell whether the request took effect. CommitThenClose bool + // CutAfter (archive downloads) sends the real answer but drops the + // connection after this many body bytes. + CutAfter int64 + // StallAfter (archive downloads) sends this many body bytes, then + // nothing until the client gives up. + StallAfter int64 + // IgnoreRange (archive downloads) answers 200 with the whole archive + // despite a Range header. + IgnoreRange bool } // Key is an API key the fake accepts. @@ -97,6 +107,8 @@ type Event struct { Resolved int64 // folder the asset landed in Key string Query string + // Range is the Range header of an archive download. + Range string } // Server is the fake. Create it with New. @@ -119,6 +131,9 @@ type Server struct { UploadExpiresIn time.Duration // Hook, when set, runs at the start of every request (outside the lock). Hook func(route Route, r *http.Request) + // LegacyExportConflicts answers export conflicts the way servers did + // before error codes: a 409 with only a message. + LegacyExportConflicts bool mu sync.Mutex seq int64 @@ -133,6 +148,12 @@ type Server struct { violations []string inflightPut int maxPut int + + exports map[int64]*fakeExport + nextExport int64 + exportTemplate []ExportDomain + exportsPaused bool + urlGeneration int } // New starts a fake and closes it when the test ends. The test fails if a @@ -149,8 +170,16 @@ func New(t testing.TB) *Server { api.HandleFunc("/api/assets", s.handleCreate) api.HandleFunc("/api/assets/", s.handleComplete) api.HandleFunc("/api/folders", s.handleFolders) + api.HandleFunc("/api/asset-exports", s.handleExports) + api.HandleFunc("/api/asset-exports/", s.handleExport) s.API = httptest.NewServer(api) - s.S3 = httptest.NewServer(http.HandlerFunc(s.handlePut)) + s.S3 = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/exports/") { + s.handleDownload(w, r) + return + } + s.handlePut(w, r) + })) s.URL = s.API.URL + "/api" t.Cleanup(func() { s.API.Close() From fd8617922de8fc7100a420b16b7820bf6a567f1e Mon Sep 17 00:00:00 2001 From: selul Date: Wed, 4 Nov 2026 15:05:43 +0200 Subject: [PATCH 2/9] export start: follow the newest export on a bare 409 below the limit Live on staging (no error codes yet) a tiny export finished between the 409 and the look at the list, so the CLI saw no running export and gave up. Below the 5-export limit the only conflict is the newest export, so it is followed even when it has finished; a coded export_limit_reached is never mistaken for it. Co-Authored-By: Claude Opus 5.5 --- internal/cli/export.go | 6 +++-- internal/cli/export_test.go | 26 ++++++++++++++++++++++ internal/cli/optimole-export-1/export.json | 13 +++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 internal/cli/optimole-export-1/export.json diff --git a/internal/cli/export.go b/internal/cli/export.go index aeaca67..c40f3ca 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -234,12 +234,14 @@ func startExport(ctx context.Context, env Env, sess *session) (int64, bool, int) return apiErr.ExportID.Value, false, ExitOK } // Servers without error codes answer both conflicts with a bare 409: - // the list says which one it is. + // the list says which one it is. Below the limit the conflict can + // only be the newest export, running then (it may have finished + // since). exports, listErr := listAllExports(ctx, sess.client) if listErr != nil { return 0, false, exportError(env, sess, name, err) } - if len(exports) > 0 && !api.IsFinalStatus(exports[0].Status) { + if apiErr.Code != api.CodeExportLimitReached && len(exports) > 0 && (!api.IsFinalStatus(exports[0].Status) || len(exports) < exportLimit) { return exports[0].ID, false, ExitOK } fmt.Fprintf(env.Stderr, "optimole %s: %s\n", name, sess.redact.Error(err)) diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go index 2545017..974f974 100644 --- a/internal/cli/export_test.go +++ b/internal/cli/export_test.go @@ -128,6 +128,32 @@ func TestExportStartFollowsTheRunningExport(t *testing.T) { } } +func TestExportStartFollowsAnExportThatFinishedAfterTheConflict(t *testing.T) { + h := newHarness(t).withKey() + // An older server's bare 409 for a running export, which finished + // before the CLI looked at the list. + h.srv.AddExport("completed", testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{archive(t, "a")}}) + h.srv.Inject(testserver.RouteExportCreate, testserver.Response{Status: 409, Body: `{"message":"Asset export is already in progress"}`}) + out := t.TempDir() + if code := h.run("export", "start", "-o", out); code != ExitOK { + t.Fatalf("exit %d, stderr %s", code, h.stderr.String()) + } + if !strings.Contains(h.stdout.String(), "Export 1 is already running; following it.") || !strings.Contains(h.stdout.String(), "Export 1 completed: 1 downloaded") { + t.Errorf("stdout: %s", h.stdout.String()) + } +} + +func TestExportStartTrustsTheLimitCode(t *testing.T) { + h := newHarness(t).withKey() + // The limit was reached, then an export was deleted before the CLI + // looked at the list: still not an export to follow. + h.srv.AddExport("completed") + h.srv.Inject(testserver.RouteExportCreate, testserver.Response{Status: 409, Body: `{"code":"export_limit_reached","message":"Maximum asset export count reached (Limit: 5)","limit":5,"count":5}`}) + if code := h.run("export", "start"); code != ExitFailure || !strings.Contains(h.stderr.String(), `"optimole export delete ID"`) { + t.Fatalf("exit %d, stderr %s", code, h.stderr.String()) + } +} + func TestExportStartAtTheLimit(t *testing.T) { for _, legacy := range []bool{false, true} { h := newHarness(t).withKey() diff --git a/internal/cli/optimole-export-1/export.json b/internal/cli/optimole-export-1/export.json new file mode 100644 index 0000000..ed7a578 --- /dev/null +++ b/internal/cli/optimole-export-1/export.json @@ -0,0 +1,13 @@ +{ + "id": 1, + "status": "completed", + "created_at": "2026-11-04T10:01:00.000000Z", + "domains": [ + { + "domain": "directUpload", + "folder": "directUpload", + "status": "completed", + "archives": [] + } + ] +} From 54e0b1c31adf0193ca90f712e129899b135b3ccd Mon Sep 17 00:00:00 2001 From: selul Date: Wed, 4 Nov 2026 15:26:47 +0200 Subject: [PATCH 3/9] export: review fixes (links, checksums, lock, cancellation, names) - Every file of an archive is read back with its checksum before the archive is put in place, so a resume that joined two same-size versions is caught and downloaded again from scratch. - Links planted in the export folder are not followed: domain folders must be plain folders, a link at a partial file is removed and the file opened with O_NOFOLLOW where available, export.json is written to a fresh O_EXCL file. - One command at a time per export folder: an OS lock (flock, LockFileEx) on .optimole-export.lock; the manifest is written only under it. - A failed look at the export cancels the downloads in flight instead of waiting for them; their partial files stay for the next run. - An archive whose size neither the export nor storage gives is refused. - Folder names are capped in bytes, not runes. - A --domain rerun keeps earlier downloads of other domains in export.json. - Ctrl-C during the conflict lookup of "export start" exits 130. - README: event order, --no-wait exit code, checksum and lock notes. Co-Authored-By: Claude Opus 5.5 --- README.md | 21 +- internal/cli/export.go | 13 +- internal/cli/export_test.go | 46 +++ .../optimole-export-1/.optimole-export.lock | 0 internal/export/download.go | 61 ++-- internal/export/export.go | 20 +- internal/export/export_test.go | 272 +++++++++++++++++- internal/export/fs.go | 70 +++++ internal/export/fs_other.go | 11 + internal/export/fs_unix.go | 22 ++ internal/export/fs_windows.go | 35 +++ internal/export/manifest.go | 30 +- internal/export/names.go | 21 +- internal/testserver/exports.go | 20 ++ internal/testserver/testserver.go | 5 + 15 files changed, 590 insertions(+), 57 deletions(-) create mode 100644 internal/cli/optimole-export-1/.optimole-export.lock create mode 100644 internal/export/fs.go create mode 100644 internal/export/fs_other.go create mode 100644 internal/export/fs_unix.go create mode 100644 internal/export/fs_windows.go diff --git a/README.md b/README.md index af947b1..fbd6389 100644 --- a/README.md +++ b/README.md @@ -295,9 +295,13 @@ downloaded. It holds no download URLs. The archives are not unpacked. - While the export runs, the CLI checks it every 5 seconds (less often after a few minutes) and downloads each archive as soon as the server lists it. - An archive is written to `images-N.zip..part` and renamed into place only when its size matches what - the server listed and it opens as a ZIP. A download that gets cut is resumed where it stopped; one that - stalls for 2 minutes is restarted. An archive that is not a valid ZIP is downloaded once more, then reported - as failed. + the server listed and every file in it reads back with its checksum. A download that gets cut is resumed + where it stopped; one that stalls for 2 minutes is restarted. An archive that fails the check (a resume that + joined two versions of a rewritten archive, for example) is downloaded once more from scratch, then reported + as failed. An archive whose size neither the server nor storage gives is not downloaded. +- One command at a time downloads into an export folder: it holds a lock on `.optimole-export.lock`, and a + second command into the same folder stops with an error. Links planted inside the export folder are not + followed. - Running `download` again skips the archives already in place and resumes the partial ones. If the server listed an archive with a new size (it rewrote it), that archive is downloaded again. - Download URLs are valid for a day; when storage refuses one, the CLI asks the server for a fresh one. @@ -309,9 +313,9 @@ downloaded. It holds no download URLs. The archives are not unpacked. ```text Started export 42. -Export 42: running example.com: running directUpload: running +Export 42: running downloading optimole-export-42/example.com/images-1.zip (1.0 GB) downloaded optimole-export-42/example.com/images-1.zip (1.0 GB in 41s) directUpload: completed, no files @@ -325,15 +329,16 @@ With `--json`: ```json {"event":"started","id":42,"created":true} -{"event":"export","id":42,"status":"running"} {"event":"domain","domain":"example.com","status":"running","archives":0} +{"event":"export","id":42,"status":"running"} {"event":"downloading","domain":"example.com","part":1,"path":"optimole-export-42/example.com/images-1.zip","size":1073741824} {"event":"archive","domain":"example.com","part":1,"path":"optimole-export-42/example.com/images-1.zip","status":"downloaded","size":1073741824,"seconds":41} {"summary":{"id":42,"status":"completed","dir":"optimole-export-42","downloaded":2,"present":0,"failed":0,"bytes":1932735283,"failed_domains":[],"interrupted":false,"timed_out":false}} ``` -An archive's `status` is `downloaded`, `present` (already on disk) or `failed` (with `error`). `created` is -`false` when the command follows an export that was already running. +Each look at the export reports its domains first, then the export's own status. An archive's `status` is +`downloaded`, `present` (already on disk) or `failed` (with `error`). `created` is `false` when the command +follows an export that was already running. An export is `queued`, `running`, `completed`, `failed` or `cancelled`. Once every domain has finished, it is `failed` if one of them failed; the other domains' archives are still downloaded, and the command exits 1. An account keeps at most 5 @@ -356,7 +361,7 @@ Prints the version, commit and build date. | Code | Meaning | |---|---| -| 0 | Every selected file was uploaded (skipped files do not count). For `export start` and `export download`: the export completed and every selected archive is on disk. | +| 0 | Every selected file was uploaded (skipped files do not count). For `export start` and `export download`: the export completed and every selected archive is on disk (`export start --no-wait`: the export was started, or one was already running). | | 1 | At least one file failed, there was nothing to upload, or the output could not be written. For exports: the export failed or was cancelled, an archive failed, `--timeout` passed, or the account keeps the most exports it may. | | 2 | Usage or configuration error (bad flags, a directory without `-r`, an unknown `--domain`, missing key, non-https URL). Nothing was uploaded or downloaded. | | 130 | Interrupted with Ctrl-C. | diff --git a/internal/cli/export.go b/internal/cli/export.go index c40f3ca..0cbb709 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -239,7 +239,7 @@ func startExport(ctx context.Context, env Env, sess *session) (int64, bool, int) // since). exports, listErr := listAllExports(ctx, sess.client) if listErr != nil { - return 0, false, exportError(env, sess, name, err) + return 0, false, exportError(env, sess, name, firstCancel(ctx, err)) } if apiErr.Code != api.CodeExportLimitReached && len(exports) > 0 && (!api.IsFinalStatus(exports[0].Status) || len(exports) < exportLimit) { return exports[0].ID, false, ExitOK @@ -256,7 +256,16 @@ func startExport(ctx context.Context, env Env, sess *session) (int64, bool, int) return exports[0].ID, false, ExitOK } } - return 0, false, exportError(env, sess, name, err) + return 0, false, exportError(env, sess, name, firstCancel(ctx, err)) +} + +// firstCancel returns the context's error when it ended (so Ctrl-C during +// a fallback lookup still exits 130), else err. +func firstCancel(ctx context.Context, err error) error { + if ctx.Err() != nil { + return ctx.Err() + } + return err } func printLimitHint(env Env, exports []api.Export) { diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go index 974f974..9b170f5 100644 --- a/internal/cli/export_test.go +++ b/internal/cli/export_test.go @@ -4,6 +4,8 @@ import ( "archive/zip" "bytes" "context" + "fmt" + "net/http" "os" "path/filepath" "strings" @@ -382,3 +384,47 @@ func TestExportMissingExport(t *testing.T) { t.Errorf("stderr: %s", h.stderr.String()) } } + +func TestExportStartInterruptedDuringTheConflictLookup(t *testing.T) { + for _, lostAnswer := range []bool{false, true} { + h := newHarness(t).withKey() + h.srv.LegacyExportConflicts = true + h.srv.AddExport("running") + if lostAnswer { + h.srv.Inject(testserver.RouteExportCreate, testserver.Response{CommitThenClose: true}) + } + ctx, cancel := context.WithCancel(context.Background()) + h.srv.Hook = func(route testserver.Route, _ *http.Request) { + if route == testserver.RouteExportList { + cancel() + } + } + if code := h.runCtx(ctx, "export", "start"); code != ExitInterrupted { + t.Errorf("lostAnswer=%v: exit %d, stderr %s", lostAnswer, code, h.stderr.String()) + } + } +} + +func TestExportListFollowsCursors(t *testing.T) { + h := newHarness(t).withKey() + for i := 0; i < exportsPerPage+3; i++ { + h.srv.AddExport("completed") + } + if code := h.run("export", "list", "--json"); code != ExitOK { + t.Fatalf("exit %d", code) + } + if n := len(h.jsonLines()); n != exportsPerPage+3 || h.srv.Count(testserver.RouteExportList) != 2 { + t.Errorf("%d exports in %d pages", n, h.srv.Count(testserver.RouteExportList)) + } + + // A server that hands out the same cursor again is not followed forever. + g := newHarness(t).withKey() + page := `{"data":[{"id":%d,"status":"completed","created_at":"2026-11-04T10:00:00.000000Z","domains":[]}],"meta":{"next_cursor":"same","has_more":true}}` + g.srv.Inject(testserver.RouteExportList, testserver.Response{Status: 200, Body: fmt.Sprintf(page, 2)}, testserver.Response{Status: 200, Body: fmt.Sprintf(page, 1)}) + if code := g.run("export", "list", "--json"); code != ExitOK { + t.Fatalf("exit %d", code) + } + if n := len(g.jsonLines()); n != 2 || g.srv.Count(testserver.RouteExportList) != 2 { + t.Errorf("%d exports in %d pages", n, g.srv.Count(testserver.RouteExportList)) + } +} diff --git a/internal/cli/optimole-export-1/.optimole-export.lock b/internal/cli/optimole-export-1/.optimole-export.lock new file mode 100644 index 0000000..e69de29 diff --git a/internal/export/download.go b/internal/export/download.go index d68d41b..c94f2e7 100644 --- a/internal/export/download.go +++ b/internal/export/download.go @@ -35,6 +35,10 @@ var errNeedURL = errors.New("the download URL was refused or the archive changed // errStalled is an attempt aborted because no bytes arrived for stallTimeout. var errStalled = errors.New("the download stalled") +// errUnknownSize is an archive whose size neither the export nor storage +// gave: without one, a download has no bound. +var errUnknownSize = errors.New("neither the export nor storage gave the archive's size") + // writeError is a failure writing to the local disk; retrying will not help. type writeError struct{ err error } @@ -60,23 +64,25 @@ type jobResult struct { err error } -// onDisk reports whether the archive is already in place: a file of the -// listed size (only checked archives are renamed into place). +// onDisk reports whether the archive is already in place: a plain file of +// the listed size (only checked archives are renamed into place). func onDisk(j job) bool { if j.size < 0 { return false } - st, err := os.Stat(j.path) + st, err := os.Lstat(j.path) return err == nil && st.Mode().IsRegular() && st.Size() == j.size } // download puts one archive at j.path. The bytes go to a ".part" file named // after the expected size, so a later run resumes only an archive of the same -// size; the file is renamed into place once its size matches and it opens as -// a ZIP. A broken ZIP is downloaded once more from scratch. +// size; the file is renamed into place once its size matches and every file +// in it reads back with its checksum. That check also catches a resume that +// joined two versions of the same size. A broken ZIP is downloaded once more +// from scratch. func (r *runner) download(ctx context.Context, j job) jobResult { - if err := os.MkdirAll(filepath.Dir(j.path), 0o755); err != nil { - return jobResult{err: &writeError{err}} + if err := realDir(filepath.Dir(j.path)); err != nil { + return jobResult{err: err} } tmp := tempPath(j) removeStaleTemps(j.path, tmp) @@ -158,7 +164,10 @@ func (r *runner) fetch(ctx context.Context, j job, tmp string, res *jobResult) e // fetchOnce makes one GET and appends the body to tmp. It returns the bytes // written. func (r *runner) fetchOnce(ctx context.Context, j job, tmp string) (int64, error) { - f, err := os.OpenFile(tmp, os.O_RDWR|os.O_CREATE, 0o644) + if err := clearLink(tmp); err != nil { + return 0, &writeError{err} + } + f, err := os.OpenFile(tmp, os.O_RDWR|os.O_CREATE|noFollow, 0o644) if err != nil { return 0, &writeError{err} } @@ -198,14 +207,14 @@ func (r *runner) fetchOnce(ctx context.Context, j job, tmp string) (int64, error if want < 0 { want = obj.Size } + if want < 0 { + return 0, errUnknownSize + } stall := time.AfterFunc(stallTimeout, cancel) defer stall.Stop() - var src io.Reader = &stallReader{r: obj.Body, timer: stall} - if want >= 0 { - // One byte more than expected shows an archive that grew. - src = io.LimitReader(src, want-offset+1) - } + // One byte more than expected shows an archive that grew. + src := io.LimitReader(&stallReader{r: obj.Body, timer: stall}, want-offset+1) w := &fileWriter{f: f} n, err := io.Copy(w, src) switch { @@ -218,9 +227,9 @@ func (r *runner) fetchOnce(ctx context.Context, j job, tmp string) (int64, error } total := offset + n switch { - case want >= 0 && total > want: + case total > want: return n, fmt.Errorf("storage sent more than the listed %d bytes: %w", want, errNeedURL) - case want >= 0 && total < want: + case total < want: return n, fmt.Errorf("%s: the archive ended after %d of %d bytes", api.OpDownloadArchive, total, want) } if err := f.Sync(); err != nil { @@ -250,7 +259,7 @@ func classifyStorageError(err error, f *os.File) error { // retryable reports whether another attempt may succeed. func retryable(err error) bool { var we *writeError - if errors.As(err, &we) || errors.Is(err, errNeedURL) { + if errors.As(err, &we) || errors.Is(err, errNeedURL) || errors.Is(err, errUnknownSize) { return false } var se *api.StorageError @@ -260,8 +269,9 @@ func retryable(err error) bool { return true } -// checkArchive checks the downloaded file's size and that it opens as a ZIP -// (its central directory reads), and returns the size. +// checkArchive checks the downloaded file's size and that it is a ZIP whose +// every file reads back with its checksum, and returns the size. Files in a +// compression format the reader lacks are not read. func checkArchive(path string, want int64) (int64, error) { st, err := os.Stat(path) if err != nil { @@ -274,7 +284,20 @@ func checkArchive(path string, want int64) (int64, error) { if err != nil { return 0, fmt.Errorf("the downloaded archive is not a valid ZIP: %w", err) } - zr.Close() + defer zr.Close() + for _, f := range zr.File { + rc, err := f.Open() + if errors.Is(err, zip.ErrAlgorithm) { + continue + } + if err == nil { + _, err = io.Copy(io.Discard, rc) + rc.Close() + } + if err != nil { + return 0, fmt.Errorf("the downloaded archive is not a valid ZIP: %q: %w", f.Name, err) + } + } return st.Size(), nil } diff --git a/internal/export/export.go b/internal/export/export.go index 4c7b626..78fcb57 100644 --- a/internal/export/export.go +++ b/internal/export/export.go @@ -99,7 +99,8 @@ type Outcome struct { Bytes int64 // Err says why the run stopped before the export finished and every // archive was tried: a failed look at the export, an *UnknownDomainError, - // the context, or the manifest that could not be written. + // another command downloading into Dir, the context, or the manifest that + // could not be written. Err error } @@ -160,6 +161,9 @@ type runner struct { parts map[partKey]*part queue []*part bytes int64 + // unlock releases the lock on Dir, held from the first look at the + // export on. + unlock func() } // Run follows the export until it is final and every selected archive it @@ -194,26 +198,29 @@ func Run(ctx context.Context, o Options) Outcome { out.Present++ } } - var unknown *UnknownDomainError - if r.exp != nil && !errors.As(out.Err, &unknown) { + if r.unlock != nil { if err := writeManifest(o.Dir, r.exp, r.folders.names, r.parts); err != nil && out.Err == nil { out.Err = err } + r.unlock() } return out } // loop is the coordinator: it looks at the export, starts downloads and // handles their results, all on one goroutine. -func (r *runner) loop(ctx context.Context) Outcome { +func (r *runner) loop(parent context.Context) Outcome { var out Outcome + ctx, cancel := context.WithCancel(parent) started := time.Now() results := make(chan finished) pollDue := make(chan struct{}, 1) pollNow, pollScheduled := true, false running := 0 defer func() { - // Wait for the downloads in flight; the context ends them early. + // Stop the downloads in flight, which keep their partial files for + // the next run, and wait for them. + cancel() for ; running > 0; running-- { r.finish(ctx, <-results) } @@ -226,6 +233,9 @@ func (r *runner) loop(ctx context.Context) Outcome { if err == nil { err = r.apply(exp) } + if err == nil && r.unlock == nil { + r.unlock, err = lockDir(r.opts.Dir) + } if err != nil { if ctx.Err() != nil { err = ctx.Err() diff --git a/internal/export/export_test.go b/internal/export/export_test.go index a17b300..e01aec5 100644 --- a/internal/export/export_test.go +++ b/internal/export/export_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -15,6 +16,7 @@ import ( "sync" "testing" "time" + "unicode/utf8" "github.com/Codeinwp/optimole-cli/internal/api" "github.com/Codeinwp/optimole-cli/internal/testserver" @@ -429,8 +431,8 @@ func TestRunKeepsHostileDomainNamesInsideTheFolder(t *testing.T) { seen[strings.ToLower(name)] = true } entries, _ := os.ReadDir(f.dir) - if len(entries) != len(domains)+1 { // + export.json - t.Errorf("%d entries in the export folder, want %d", len(entries), len(domains)+1) + if len(entries) != len(domains)+2 { // + export.json and the lock + t.Errorf("%d entries in the export folder, want %d", len(entries), len(domains)+2) } if _, err := os.Stat(filepath.Join(filepath.Dir(f.dir), "evil")); !os.IsNotExist(err) { t.Errorf("wrote outside the export folder: %v", err) @@ -456,8 +458,14 @@ func TestFolderNames(t *testing.T) { t.Errorf("%q -> %q, want %q", d, m.names[d], name) } } - if got := m.names[strings.Repeat("a", 300)]; len(got) != maxFolderRunes { - t.Errorf("long name kept %d characters", len(got)) + if got := m.names[strings.Repeat("a", 300)]; len(got) != maxFolderBytes { + t.Errorf("long name kept %d bytes", len(got)) + } + // The cap counts bytes: 50 three-byte runes would be 150. + wide := strings.Repeat("界", 50) + ".com" + m.add([]string{wide}) + if got := m.names[wide]; len(got) > maxFolderBytes || !utf8.ValidString(got) || !strings.HasPrefix(got, "界界") { + t.Errorf("multibyte name -> %q (%d bytes)", got, len(got)) } // Domains seen later never take an earlier domain's folder. m.add([]string{"EXAMPLE.COM"}) @@ -622,3 +630,259 @@ func TestFetchNamesAStalledDownload(t *testing.T) { t.Fatalf("got %d bytes, %v", n, err) } } + +// storedZip builds an uncompressed ZIP, so contents of the same length give +// archives of the same size. +func storedZip(t *testing.T, name, content string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store, Modified: time.Date(2026, 11, 4, 0, 0, 0, 0, time.UTC)}) + if err != nil { + t.Fatal(err) + } + io.WriteString(w, content) + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestRunDetectsAResumeAcrossASameSizeRewrite(t *testing.T) { + f := newFixture(t) + old := storedZip(t, "a.jpg", strings.Repeat("o", 300)) + rewritten := storedZip(t, "a.jpg", strings.Repeat("n", 300)) + if len(old) != len(rewritten) { + t.Fatal("fixture archives differ in size") + } + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{old}}) + // The first download is cut inside the file data; the archive is then + // rewritten with the same size, so the resume joins two versions. + f.srv.Inject(testserver.RouteDownload, testserver.Response{CutAfter: 100}) + var once sync.Once + f.srv.Hook = func(route testserver.Route, r *http.Request) { + if route == testserver.RouteDownload && r.Header.Get("Range") != "" { + once.Do(func() { f.srv.RewriteArchive(id, "example.com", 1, rewritten) }) + } + } + out := f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != 1 { + t.Fatalf("outcome: %+v", out) + } + if !bytes.Equal(f.file("example.com/images-1.zip"), rewritten) { + t.Error("kept a mix of two versions") + } + var ranges []string + for _, d := range f.downloads() { + ranges = append(ranges, d.Range) + } + if strings.Join(ranges, "|") != "|bytes=100-|" { + t.Errorf("ranges %q: want a resume, then a fresh download", ranges) + } +} + +func TestDownloadRefusesAnArchiveOfUnknownSize(t *testing.T) { + requests := 0 + storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() // chunked: no Content-Length + w.Write(bytes.Repeat([]byte("x"), 1<<16)) + })) + defer storage.Close() + f := newFixture(t) + path := filepath.Join(f.dir, "example.com", "images-1.zip") + res := f.newRunner().download(context.Background(), job{path: path, url: storage.URL + "/x", size: -1}) + if !errors.Is(res.err, errUnknownSize) || requests != 1 || res.written != 0 { + t.Fatalf("got %+v after %d requests", res, requests) + } +} + +func TestRunCancelsDownloadsWhenTheExportCannotBeRead(t *testing.T) { + f := newFixture(t) + f.srv.SetExportDomains(testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{zipOf(t, "a"), zipOf(t, "b")}}) + exp, _ := f.client.CreateExport(context.Background()) + // The first download stalls (for good: the stall timeout is minutes); + // meanwhile the export stays running and every look at it fails. + f.srv.Inject(testserver.RouteDownload, testserver.Response{StallAfter: 20}) + var once sync.Once + gets := 0 + f.srv.Hook = func(route testserver.Route, _ *http.Request) { + if route == testserver.RouteExportGet { + // The second look listed the first archive; the export stays + // running from the third on. + f.mu.Lock() + if gets++; gets == 3 { + f.srv.PauseExports(true) + } + f.mu.Unlock() + } + if route == testserver.RouteDownload { + once.Do(func() { + var errs []testserver.Response + for i := 0; i < 20; i++ { + errs = append(errs, testserver.Response{Status: http.StatusInternalServerError, Body: `{"message":"Server Error"}`}) + } + f.srv.Inject(testserver.RouteExportGet, errs...) + }) + } + } + done := make(chan Outcome, 1) + go func() { done <- f.run(context.Background(), exp.ID) }() + select { + case out := <-done: + var apiErr *api.Error + if !errors.As(out.Err, &apiErr) || apiErr.StatusCode != http.StatusInternalServerError || out.Failed != 0 { + t.Fatalf("outcome: %+v", out) + } + case <-time.After(10 * time.Second): + t.Fatal("the run waited for the stalled download") + } + parts, _ := filepath.Glob(filepath.Join(f.dir, "example.com", "*"+partSuffix)) + if len(parts) != 1 { + t.Errorf("partial files: %v", parts) + } +} + +func TestRunResumesDownloadsCancelledMidway(t *testing.T) { + f := newFixture(t) + a, b := zipOf(t, "a"), zipOf(t, "b") + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{a, b}}) + f.srv.Inject(testserver.RouteDownload, testserver.Response{StallAfter: 30}, testserver.Response{StallAfter: 40}) + ctx, cancel := context.WithCancel(context.Background()) + stalled := 0 + f.srv.Hook = func(route testserver.Route, _ *http.Request) { + if route == testserver.RouteDownload { + f.mu.Lock() + stalled++ + if stalled == 2 { + time.AfterFunc(100*time.Millisecond, cancel) + } + f.mu.Unlock() + } + } + out := f.run(ctx, id) + if !errors.Is(out.Err, context.Canceled) || out.Failed != 0 || out.Downloaded != 0 { + t.Fatalf("cancelled run: %+v", out) + } + parts, _ := filepath.Glob(filepath.Join(f.dir, "example.com", "*"+partSuffix)) + if len(parts) != 2 { + t.Fatalf("partial files: %v", parts) + } + + f.srv.Hook = nil + before := len(f.downloads()) + out = f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != 2 { + t.Fatalf("resumed run: %+v", out) + } + if !bytes.Equal(f.file("example.com/images-1.zip"), a) || !bytes.Equal(f.file("example.com/images-2.zip"), b) { + t.Error("resumed archives differ") + } + for _, d := range f.downloads()[before:] { + if d.Range != "bytes=30-" && d.Range != "bytes=40-" { + t.Errorf("resume without the right range: %+v", d) + } + } +} + +func TestRunLimitsParallelDownloads(t *testing.T) { + f := newFixture(t) + var archives [][]byte + for i := 0; i < 7; i++ { + archives = append(archives, zipOf(t, fmt.Sprintf("f%d", i))) + } + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: archives}) + f.srv.DownloadDelay = 30 * time.Millisecond + out := Run(context.Background(), Options{Client: f.client, ExportID: id, Dir: f.dir, Concurrency: 3, Sleep: immediate}) + if out.Err != nil || out.Downloaded != 7 { + t.Fatalf("outcome: %+v", out) + } + if n := f.srv.MaxConcurrentDownloads(); n != 3 { + t.Errorf("at most %d downloads at once, want 3", n) + } +} + +func TestRunRefusesAFolderAnotherRunHolds(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{zipOf(t, "a")}}) + unlock, err := lockDir(f.dir) + if err != nil { + t.Fatal(err) + } + defer unlock() + out := f.run(context.Background(), id) + if !errors.Is(out.Err, errLocked) || len(f.downloads()) != 0 { + t.Fatalf("outcome %+v after %d downloads", out, len(f.downloads())) + } + if _, err := os.Stat(filepath.Join(f.dir, ManifestName)); !os.IsNotExist(err) { + t.Errorf("wrote the manifest of a folder another run holds: %v", err) + } + unlock() + if out := f.run(context.Background(), id); out.Err != nil || out.Downloaded != 1 { + t.Errorf("after the lock was released: %+v", out) + } +} + +func TestRunManifestKeepsArchivesOfUnselectedDomains(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, + testserver.ExportDomain{Domain: "a.com", Archives: [][]byte{zipOf(t, "a")}}, + testserver.ExportDomain{Domain: "b.com", Archives: [][]byte{zipOf(t, "b")}}, + ) + f.run(context.Background(), id) + f.run(context.Background(), id, "a.com") + var m manifest + if err := json.Unmarshal(f.file(ManifestName), &m); err != nil { + t.Fatal(err) + } + for _, d := range m.Domains { + if !d.Archives[0].Downloaded { + t.Errorf("%s marked not downloaded after a filtered rerun", d.Domain) + } + } +} + +func TestRunRefusesLinksInTheExportFolder(t *testing.T) { + outside := t.TempDir() + victim := filepath.Join(outside, "victim.txt") + link := func(t *testing.T, target, name string) { + t.Helper() + if err := os.Symlink(target, name); err != nil { + t.Skipf("cannot create symlinks here: %v", err) + } + } + data := zipOf(t, "a.jpg") + t.Run("domain folder", func(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + os.MkdirAll(f.dir, 0o755) + link(t, outside, filepath.Join(f.dir, "example.com")) + out := f.run(context.Background(), id) + if out.Failed != 1 { + t.Fatalf("outcome: %+v", out) + } + if entries, _ := os.ReadDir(outside); len(entries) != 0 { + t.Errorf("wrote through the link: %v", entries) + } + }) + t.Run("partial file and manifest", func(t *testing.T) { + os.WriteFile(victim, []byte("keep me"), 0o644) + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + final := filepath.Join(f.dir, "example.com", "images-1.zip") + os.MkdirAll(filepath.Dir(final), 0o755) + link(t, victim, tempPath(job{path: final, size: int64(len(data))})) + link(t, victim, filepath.Join(f.dir, ManifestName+partSuffix)) + out := f.run(context.Background(), id) + if out.Err != nil || out.Downloaded != 1 { + t.Fatalf("outcome: %+v", out) + } + if got, _ := os.ReadFile(victim); string(got) != "keep me" { + t.Errorf("the link's target was written: %q", got) + } + if !bytes.Equal(f.file("example.com/images-1.zip"), data) { + t.Error("archive differs") + } + }) +} diff --git a/internal/export/fs.go b/internal/export/fs.go new file mode 100644 index 0000000..2ea5710 --- /dev/null +++ b/internal/export/fs.go @@ -0,0 +1,70 @@ +package export + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" +) + +// lockName is the file a run locks in the export folder, so two commands +// never write the same archives or manifest at once. +const lockName = ".optimole-export.lock" + +// errLocked means another command holds the export folder's lock. +var errLocked = errors.New("another optimole command is downloading into this folder") + +// lockDir creates dir as a plain folder and locks it for this process. The +// lock goes with the returned func, or with the process. +func lockDir(dir string) (func(), error) { + if err := realDir(dir); err != nil { + return nil, err + } + path := filepath.Join(dir, lockName) + if err := clearLink(path); err != nil { + return nil, &writeError{err} + } + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|noFollow, 0o644) + if err != nil { + return nil, &writeError{err} + } + if err := lockFile(f); err != nil { + f.Close() + return nil, fmt.Errorf("%s: %w", dir, err) + } + return func() { f.Close() }, nil +} + +// realDir creates dir and its parents, and checks that dir is a plain folder: +// a symlink or another reparse point planted there would send the archives +// somewhere else. Lstat reports those as links or irregular files, never as +// folders. +func realDir(dir string) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return &writeError{err} + } + st, err := os.Lstat(dir) + if err != nil { + return &writeError{err} + } + if !st.IsDir() { + return &writeError{fmt.Errorf("%s is not a plain folder (a link?); refusing to write through it", dir)} + } + return nil +} + +// clearLink removes path when it is a link or another non-regular file, so +// the open that follows creates a plain file; a link's target is left alone. +func clearLink(path string) error { + st, err := os.Lstat(path) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil + case err != nil: + return err + case st.Mode().IsRegular(): + return nil + } + return os.Remove(path) +} diff --git a/internal/export/fs_other.go b/internal/export/fs_other.go new file mode 100644 index 0000000..4cedc93 --- /dev/null +++ b/internal/export/fs_other.go @@ -0,0 +1,11 @@ +//go:build !(darwin || dragonfly || freebsd || linux || netbsd || openbsd || windows) + +package export + +import "os" + +// noFollow is not available: callers check with Lstat before opening. +const noFollow = 0 + +// lockFile does nothing here: there is no portable lock. +func lockFile(*os.File) error { return nil } diff --git a/internal/export/fs_unix.go b/internal/export/fs_unix.go new file mode 100644 index 0000000..6c9aca0 --- /dev/null +++ b/internal/export/fs_unix.go @@ -0,0 +1,22 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd + +package export + +import ( + "errors" + "os" + "syscall" +) + +// noFollow makes an open fail on a symlink in the last path component. +const noFollow = syscall.O_NOFOLLOW + +// lockFile takes an exclusive lock on f, released when f is closed (or the +// process ends). It fails with errLocked when another process holds it. +func lockFile(f *os.File) error { + err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if errors.Is(err, syscall.EWOULDBLOCK) { + return errLocked + } + return err +} diff --git a/internal/export/fs_windows.go b/internal/export/fs_windows.go new file mode 100644 index 0000000..24567f9 --- /dev/null +++ b/internal/export/fs_windows.go @@ -0,0 +1,35 @@ +//go:build windows + +package export + +import ( + "errors" + "os" + "syscall" + "unsafe" +) + +// noFollow is not available: callers check with Lstat before opening. +const noFollow = 0 + +const ( + lockfileFailImmediately = 0x1 + lockfileExclusiveLock = 0x2 + errorLockViolation = syscall.Errno(33) +) + +var procLockFileEx = syscall.NewLazyDLL("kernel32.dll").NewProc("LockFileEx") + +// lockFile takes an exclusive lock on f, released when f is closed (or the +// process ends). It fails with errLocked when another handle holds it. +func lockFile(f *os.File) error { + var ol syscall.Overlapped + r, _, err := procLockFileEx.Call(f.Fd(), lockfileExclusiveLock|lockfileFailImmediately, 0, 1, 0, uintptr(unsafe.Pointer(&ol))) + if r != 0 { + return nil + } + if errors.Is(err, errorLockViolation) { + return errLocked + } + return err +} diff --git a/internal/export/manifest.go b/internal/export/manifest.go index 8fcc42f..554919d 100644 --- a/internal/export/manifest.go +++ b/internal/export/manifest.go @@ -2,7 +2,9 @@ package export import ( "encoding/json" + "errors" "fmt" + "io/fs" "os" "path/filepath" @@ -38,8 +40,8 @@ type manifestArchive struct { Downloaded bool `json:"downloaded"` } -// writeManifest writes ManifestName into dir from the export's last answer, -// replacing an earlier one in one step. +// writeManifest writes ManifestName into dir (locked by this run) from the +// export's last answer, replacing an earlier one in one step. func writeManifest(dir string, exp *api.Export, folders map[string]string, parts map[partKey]*part) error { m := manifest{ID: exp.ID, Status: exp.Status, CreatedAt: exp.CreatedAt, Domains: []manifestDomain{}} for _, d := range exp.Domains { @@ -56,8 +58,12 @@ func writeManifest(dir string, exp *api.Export, folders map[string]string, parts if lp.Size.Valid { ma.Size = lp.Size.Value } - if p := parts[partKey{d.Domain, lp.Number.Value}]; p != nil && p.state == done && (ma.Size < 0 || ma.Size == p.doneSize) { - ma.Downloaded = true + if p := parts[partKey{d.Domain, lp.Number.Value}]; p != nil { + ma.Downloaded = p.state == done && (ma.Size < 0 || ma.Size == p.doneSize) + } else { + // A domain this run did not select: an earlier run may have + // put the archive in place. + ma.Downloaded = onDisk(job{path: filepath.Join(dir, md.Folder, filepath.Base(ma.File)), size: ma.Size}) } md.Archives = append(md.Archives, ma) } @@ -67,12 +73,20 @@ func writeManifest(dir string, exp *api.Export, folders map[string]string, parts if err != nil { return err } - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("writing %s: %w", ManifestName, err) - } path := filepath.Join(dir, ManifestName) tmp := path + partSuffix - if err := os.WriteFile(tmp, append(data, '\n'), 0o644); err != nil { + // A fresh file (O_EXCL), never one a link planted there points to. + if err := os.Remove(tmp); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("writing %s: %w", ManifestName, err) + } + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err == nil { + _, err = f.Write(append(data, '\n')) + if cerr := f.Close(); err == nil { + err = cerr + } + } + if err != nil { os.Remove(tmp) return fmt.Errorf("writing %s: %w", ManifestName, err) } diff --git a/internal/export/names.go b/internal/export/names.go index 8824ea7..17fe7ad 100644 --- a/internal/export/names.go +++ b/internal/export/names.go @@ -5,10 +5,12 @@ import ( "sort" "strings" "unicode" + "unicode/utf8" ) -// maxFolderRunes keeps folder names well inside path length limits. -const maxFolderRunes = 100 +// maxFolderBytes keeps folder names, suffix included, well inside the 255 +// bytes (or UTF-16 units) a path component may have. +const maxFolderBytes = 100 // windowsReserved are device names Windows refuses as a file name, with or // without an extension. @@ -51,21 +53,18 @@ func (m *folderMap) add(domains []string) { // safeSegment turns a domain name into a file name that is valid and inert on // every platform: letters, digits, ".", "-" and "_" only (anything else -// becomes "_"), at most maxFolderRunes long, no leading or trailing dot, and +// becomes "_"), at most maxFolderBytes long, no leading or trailing dot, and // no Windows device name. func safeSegment(domain string) string { var b strings.Builder - n := 0 for _, r := range domain { - if n == maxFolderRunes { - break + if !(r == '.' || r == '-' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)) { + r = '_' } - if r == '.' || r == '-' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) { - b.WriteRune(r) - } else { - b.WriteRune('_') + if b.Len()+utf8.RuneLen(r) > maxFolderBytes { + break } - n++ + b.WriteRune(r) } name := strings.Trim(b.String(), ".") if name == "" { diff --git a/internal/testserver/exports.go b/internal/testserver/exports.go index a3601b5..7203200 100644 --- a/internal/testserver/exports.go +++ b/internal/testserver/exports.go @@ -102,6 +102,14 @@ func (s *Server) ExpireArchiveURLs() { s.urlGeneration++ } +// MaxConcurrentDownloads is the highest number of archive downloads seen at +// once. +func (s *Server) MaxConcurrentDownloads() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.maxDownloads +} + // ExportIDs lists the stored exports, oldest first. func (s *Server) ExportIDs() []int64 { s.mu.Lock() @@ -397,6 +405,18 @@ func (s *Server) handleDownload(w http.ResponseWriter, r *http.Request) { status := 0 var drop func() defer func() { s.finishThenDrop(ev, status, drop) }() + s.mu.Lock() + s.inflightDownloads++ + s.maxDownloads = max(s.maxDownloads, s.inflightDownloads) + s.mu.Unlock() + defer func() { + s.mu.Lock() + s.inflightDownloads-- + s.mu.Unlock() + }() + if s.DownloadDelay > 0 { + time.Sleep(s.DownloadDelay) + } if r.Header.Get("Authorization") != "" { s.violate("archive download carried an Authorization header") } diff --git a/internal/testserver/testserver.go b/internal/testserver/testserver.go index 5f0d2d7..632ae1a 100644 --- a/internal/testserver/testserver.go +++ b/internal/testserver/testserver.go @@ -126,6 +126,8 @@ type Server struct { CompleteDelay time.Duration // PutDelay slows down storage uploads. PutDelay time.Duration + // DownloadDelay slows down archive downloads. + DownloadDelay time.Duration // UploadExpiresIn sets the expires_at of new upload URLs (default 15 // minutes; negative values give URLs that are already expired). UploadExpiresIn time.Duration @@ -154,6 +156,9 @@ type Server struct { exportTemplate []ExportDomain exportsPaused bool urlGeneration int + + inflightDownloads int + maxDownloads int } // New starts a fake and closes it when the test ends. The test fails if a From 92762a59799e03a105b5d8c4e5c5399bc93dd453 Mon Sep 17 00:00:00 2001 From: selul Date: Wed, 4 Nov 2026 15:40:05 +0200 Subject: [PATCH 4/9] export: interruptible archive check, link-safe presence check, Windows resume hint - The checksum read stops when the run ends, keeping the partial file for the next run instead of holding the folder lock until it finishes. - An archive counts as already downloaded only in a plain domain folder, so a linked folder never passes for the export's; with no listed size, a plain file in place counts (only checked archives are renamed there), so a --domain rerun keeps size-less archives in export.json. - The "Resume with" hint quotes the folder for a shell: as is when plain, single quotes on Unix, double quotes on Windows (Go quoting doubled every backslash of a Windows path). - README: files in a compression format Go's ZIP reader lacks are not read. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- internal/cli/export.go | 21 ++++++- internal/cli/export_test.go | 23 ++++++- internal/export/download.go | 41 +++++++++---- internal/export/export_test.go | 99 ++++++++++++++++++++++++++----- internal/testserver/exports.go | 10 ++-- internal/testserver/testserver.go | 2 + 7 files changed, 166 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index fbd6389..010e7b4 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,8 @@ downloaded. It holds no download URLs. The archives are not unpacked. - While the export runs, the CLI checks it every 5 seconds (less often after a few minutes) and downloads each archive as soon as the server lists it. - An archive is written to `images-N.zip..part` and renamed into place only when its size matches what - the server listed and every file in it reads back with its checksum. A download that gets cut is resumed + the server listed and every file in it reads back with its checksum (files in a compression format Go's ZIP + reader lacks are not read; Optimole's archives use deflate). A download that gets cut is resumed where it stopped; one that stalls for 2 minutes is restarted. An archive that fails the check (a resume that joined two versions of a rewritten archive, for example) is downloaded once more from scratch, then reported as failed. An archive whose size neither the server nor storage gives is not downloaded. diff --git a/internal/cli/export.go b/internal/cli/export.go index 0cbb709..dab4d69 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -8,10 +8,13 @@ import ( "fmt" "net/http" "path/filepath" + "runtime" "strconv" "strings" "text/tabwriter" "time" + "unicode" + "unicode/utf8" "github.com/Codeinwp/optimole-cli/internal/api" "github.com/Codeinwp/optimole-cli/internal/export" @@ -259,6 +262,22 @@ func startExport(ctx context.Context, env Env, sess *session) (int64, bool, int) return 0, false, exportError(env, sess, name, firstCancel(ctx, err)) } +// shellQuote writes a path for a command to paste: as is when it needs no +// quotes, else in single quotes (double quotes on Windows, where a path +// cannot contain them). +func shellQuote(s string) string { + plain := s != "" && strings.IndexFunc(s, func(r rune) bool { + return !(r < utf8.RuneSelf && (unicode.IsLetter(r) || unicode.IsDigit(r)) || strings.ContainsRune(`._-/\:+,@%=`, r)) + }) < 0 + switch { + case plain: + return s + case runtime.GOOS == "windows": + return `"` + s + `"` + } + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + // firstCancel returns the context's error when it ended (so Ctrl-C during // a fallback lookup still exits 130), else err. func firstCancel(ctx context.Context, err error) error { @@ -342,7 +361,7 @@ func downloadExport(ctx, runCtx context.Context, env Env, sess *session, rep *ou if interrupted || timedOut { resume := fmt.Sprintf("optimole export download %d", id) if df.output != "." { - resume += " -o " + strconv.Quote(df.output) + resume += " -o " + shellQuote(df.output) } fmt.Fprintf(env.Stderr, "Export %d continues on the server. Resume with: %s\n", id, resume) } diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go index 9b170f5..234cc7f 100644 --- a/internal/cli/export_test.go +++ b/internal/cli/export_test.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -224,7 +225,7 @@ func TestExportDownloadStopsAndTellsHowToResume(t *testing.T) { if code := h.run("export", "download", "latest", "-o", out, "--timeout", "150ms"); code != ExitFailure { t.Fatalf("timeout: exit %d", code) } - resume := `Resume with: optimole export download 1 -o "` + out + `"` + resume := "Resume with: optimole export download 1 -o " + shellQuote(out) if !strings.Contains(h.stderr.String(), "timed out after 150ms") || !strings.Contains(h.stderr.String(), resume) { t.Errorf("timeout stderr: %s", h.stderr.String()) } @@ -428,3 +429,23 @@ func TestExportListFollowsCursors(t *testing.T) { t.Errorf("%d exports in %d pages", n, g.srv.Count(testserver.RouteExportList)) } } + +func TestShellQuote(t *testing.T) { + cases := map[string]string{ + "out": "out", + "/backups/2026-11": "/backups/2026-11", + `C:\Backups\Optimole`: `C:\Backups\Optimole`, + } + if runtime.GOOS == "windows" { + cases["my exports"] = `"my exports"` + } else { + cases["my exports"] = "'my exports'" + cases["it's"] = `'it'\''s'` + cases["$HOME"] = "'$HOME'" + } + for in, want := range cases { + if got := shellQuote(in); got != want { + t.Errorf("shellQuote(%q) = %s, want %s", in, got, want) + } + } +} diff --git a/internal/export/download.go b/internal/export/download.go index c94f2e7..83de8d0 100644 --- a/internal/export/download.go +++ b/internal/export/download.go @@ -64,22 +64,24 @@ type jobResult struct { err error } -// onDisk reports whether the archive is already in place: a plain file of -// the listed size (only checked archives are renamed into place). +// onDisk reports whether the archive is already in place: a plain file, of +// the listed size when there is one, in a plain domain folder. Only checked +// archives are renamed into place. func onDisk(j job) bool { - if j.size < 0 { + if dir, err := os.Lstat(filepath.Dir(j.path)); err != nil || !dir.IsDir() { return false } st, err := os.Lstat(j.path) - return err == nil && st.Mode().IsRegular() && st.Size() == j.size + return err == nil && st.Mode().IsRegular() && (j.size < 0 || st.Size() == j.size) } // download puts one archive at j.path. The bytes go to a ".part" file named // after the expected size, so a later run resumes only an archive of the same // size; the file is renamed into place once its size matches and every file -// in it reads back with its checksum. That check also catches a resume that -// joined two versions of the same size. A broken ZIP is downloaded once more -// from scratch. +// in it (in a compression format the reader knows) reads back with its +// checksum. That check also catches a resume that joined two versions of the +// same size. A broken ZIP is downloaded once more from scratch; an +// interrupted check keeps the partial file for the next run. func (r *runner) download(ctx context.Context, j job) jobResult { if err := realDir(filepath.Dir(j.path)); err != nil { return jobResult{err: err} @@ -93,7 +95,11 @@ func (r *runner) download(ctx context.Context, j job) jobResult { res.err = err return res } - size, err := checkArchive(tmp, j.size) + size, err := checkArchive(ctx, tmp, j.size) + if ctx.Err() != nil { + res.err = ctx.Err() + return res + } if err == nil { if err := os.Rename(tmp, j.path); err != nil { res.err = &writeError{err} @@ -271,8 +277,8 @@ func retryable(err error) bool { // checkArchive checks the downloaded file's size and that it is a ZIP whose // every file reads back with its checksum, and returns the size. Files in a -// compression format the reader lacks are not read. -func checkArchive(path string, want int64) (int64, error) { +// compression format the reader lacks are not read. It stops when ctx ends. +func checkArchive(ctx context.Context, path string, want int64) (int64, error) { st, err := os.Stat(path) if err != nil { return 0, &writeError{err} @@ -291,7 +297,7 @@ func checkArchive(path string, want int64) (int64, error) { continue } if err == nil { - _, err = io.Copy(io.Discard, rc) + _, err = io.Copy(io.Discard, ctxReader{ctx, rc}) rc.Close() } if err != nil { @@ -309,6 +315,19 @@ func backoff(failures int) time.Duration { return d } +// ctxReader stops reading once ctx ends. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} + // stallReader pushes back the stall timer whenever bytes arrive. type stallReader struct { r io.Reader diff --git a/internal/export/export_test.go b/internal/export/export_test.go index e01aec5..471e181 100644 --- a/internal/export/export_test.go +++ b/internal/export/export_test.go @@ -825,20 +825,27 @@ func TestRunRefusesAFolderAnotherRunHolds(t *testing.T) { } func TestRunManifestKeepsArchivesOfUnselectedDomains(t *testing.T) { - f := newFixture(t) - id := f.srv.AddExport(api.ExportCompleted, - testserver.ExportDomain{Domain: "a.com", Archives: [][]byte{zipOf(t, "a")}}, - testserver.ExportDomain{Domain: "b.com", Archives: [][]byte{zipOf(t, "b")}}, - ) - f.run(context.Background(), id) - f.run(context.Background(), id, "a.com") - var m manifest - if err := json.Unmarshal(f.file(ManifestName), &m); err != nil { - t.Fatal(err) - } - for _, d := range m.Domains { - if !d.Archives[0].Downloaded { - t.Errorf("%s marked not downloaded after a filtered rerun", d.Domain) + for _, sizeless := range []bool{false, true} { + f := newFixture(t) + f.srv.OmitArchiveSizes = sizeless + id := f.srv.AddExport(api.ExportCompleted, + testserver.ExportDomain{Domain: "a.com", Archives: [][]byte{zipOf(t, "a")}}, + testserver.ExportDomain{Domain: "b.com", Archives: [][]byte{zipOf(t, "b")}}, + ) + if out := f.run(context.Background(), id); out.Downloaded != 2 { + t.Fatalf("sizeless=%v: first run %+v", sizeless, out) + } + if out := f.run(context.Background(), id, "a.com"); out.Present != 1 { + t.Errorf("sizeless=%v: filtered rerun %+v", sizeless, out) + } + var m manifest + if err := json.Unmarshal(f.file(ManifestName), &m); err != nil { + t.Fatal(err) + } + for _, d := range m.Domains { + if !d.Archives[0].Downloaded { + t.Errorf("sizeless=%v: %s marked not downloaded after a filtered rerun", sizeless, d.Domain) + } } } } @@ -866,6 +873,29 @@ func TestRunRefusesLinksInTheExportFolder(t *testing.T) { t.Errorf("wrote through the link: %v", entries) } }) + t.Run("domain folder holding a matching archive", func(t *testing.T) { + f := newFixture(t) + id := f.srv.AddExport(api.ExportCompleted, + testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}, + testserver.ExportDomain{Domain: "other.com", Archives: [][]byte{data}}, + ) + elsewhere := t.TempDir() + os.WriteFile(filepath.Join(elsewhere, "images-1.zip"), data, 0o644) + os.MkdirAll(f.dir, 0o755) + link(t, elsewhere, filepath.Join(f.dir, "example.com")) + link(t, elsewhere, filepath.Join(f.dir, "other.com")) + out := f.run(context.Background(), id, "example.com") + if out.Present != 0 || out.Failed != 1 { + t.Fatalf("outcome: %+v", out) + } + var m manifest + json.Unmarshal(f.file(ManifestName), &m) + for _, d := range m.Domains { + if d.Archives[0].Downloaded { + t.Errorf("%s: an archive behind a link counted as downloaded", d.Domain) + } + } + }) t.Run("partial file and manifest", func(t *testing.T) { os.WriteFile(victim, []byte("keep me"), 0o644) f := newFixture(t) @@ -886,3 +916,44 @@ func TestRunRefusesLinksInTheExportFolder(t *testing.T) { } }) } + +// endsOnceComplete is a context that reports Canceled once path holds size +// bytes: the download completes, and the check that follows sees the end. +type endsOnceComplete struct { + context.Context + path string + size int64 +} + +func (c endsOnceComplete) Err() error { + if st, err := os.Stat(c.path); err == nil && st.Size() == c.size { + return context.Canceled + } + return nil +} + +func TestDownloadKeepsThePartialFileWhenTheCheckIsInterrupted(t *testing.T) { + f := newFixture(t) + data := zipOf(t, "a.jpg", "b.jpg") + id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{data}}) + exp, err := f.client.GetExport(context.Background(), id) + if err != nil { + t.Fatal(err) + } + j := job{path: filepath.Join(f.dir, "example.com", "images-1.zip"), url: exp.Domains[0].Parts[0].URL, size: int64(len(data))} + res := f.newRunner().download(endsOnceComplete{context.Background(), tempPath(j), j.size}, j) + if !errors.Is(res.err, context.Canceled) { + t.Fatalf("got %+v", res) + } + if st, err := os.Stat(tempPath(j)); err != nil || st.Size() != j.size { + t.Errorf("partial file: %v", err) + } + if _, err := os.Stat(j.path); !os.IsNotExist(err) { + t.Errorf("archive put in place unchecked: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := checkArchive(ctx, tempPath(j), j.size); !errors.Is(err, context.Canceled) { + t.Errorf("check on an ended context: %v", err) + } +} diff --git a/internal/testserver/exports.go b/internal/testserver/exports.go index 7203200..5f48d1f 100644 --- a/internal/testserver/exports.go +++ b/internal/testserver/exports.go @@ -198,11 +198,11 @@ func (s *Server) exportDetailLocked(e *fakeExport) map[string]any { for _, d := range e.domains { files := []map[string]any{} for i := 0; i < d.listed; i++ { - files = append(files, map[string]any{ - "part_number": i + 1, - "url": s.archiveURLLocked(e.id, d.name, i+1), - "size": len(d.archives[i]), - }) + file := map[string]any{"part_number": i + 1, "url": s.archiveURLLocked(e.id, d.name, i+1)} + if !s.OmitArchiveSizes { + file["size"] = len(d.archives[i]) + } + files = append(files, file) } domains = append(domains, map[string]any{"domain": d.name, "status": d.status, "zip_files": files}) } diff --git a/internal/testserver/testserver.go b/internal/testserver/testserver.go index 632ae1a..c035a48 100644 --- a/internal/testserver/testserver.go +++ b/internal/testserver/testserver.go @@ -136,6 +136,8 @@ type Server struct { // LegacyExportConflicts answers export conflicts the way servers did // before error codes: a 409 with only a message. LegacyExportConflicts bool + // OmitArchiveSizes lists export archives without their size. + OmitArchiveSizes bool mu sync.Mutex seq int64 From f9e974aa01bb59743a75995feb787a2c12f19460 Mon Sep 17 00:00:00 2001 From: selul Date: Wed, 4 Nov 2026 15:45:48 +0200 Subject: [PATCH 5/9] export: quote resume paths with backslashes on Unix, keep trailing backslashes on Windows Co-Authored-By: Claude Opus 5.5 --- internal/cli/export.go | 17 ++++++++++++----- internal/cli/export_test.go | 8 +++++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/cli/export.go b/internal/cli/export.go index dab4d69..162aa29 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -263,17 +263,24 @@ func startExport(ctx context.Context, env Env, sess *session) (int64, bool, int) } // shellQuote writes a path for a command to paste: as is when it needs no -// quotes, else in single quotes (double quotes on Windows, where a path -// cannot contain them). +// quotes, else in single quotes, or on Windows in double quotes (a path +// cannot contain them), where a backslash run before the closing quote is +// doubled so it does not escape the quote. A backslash needs quotes outside +// Windows. func shellQuote(s string) string { + windows := runtime.GOOS == "windows" plain := s != "" && strings.IndexFunc(s, func(r rune) bool { - return !(r < utf8.RuneSelf && (unicode.IsLetter(r) || unicode.IsDigit(r)) || strings.ContainsRune(`._-/\:+,@%=`, r)) + if r == '\\' { + return !windows + } + return !(r < utf8.RuneSelf && (unicode.IsLetter(r) || unicode.IsDigit(r)) || strings.ContainsRune("._-/:+,@%=", r)) }) < 0 switch { case plain: return s - case runtime.GOOS == "windows": - return `"` + s + `"` + case windows: + trailing := len(s) - len(strings.TrimRight(s, `\`)) + return `"` + s + strings.Repeat(`\`, trailing) + `"` } return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go index 234cc7f..f275c1f 100644 --- a/internal/cli/export_test.go +++ b/internal/cli/export_test.go @@ -432,16 +432,18 @@ func TestExportListFollowsCursors(t *testing.T) { func TestShellQuote(t *testing.T) { cases := map[string]string{ - "out": "out", - "/backups/2026-11": "/backups/2026-11", - `C:\Backups\Optimole`: `C:\Backups\Optimole`, + "out": "out", + "/backups/2026-11": "/backups/2026-11", } if runtime.GOOS == "windows" { + cases[`C:\Backups\Optimole`] = `C:\Backups\Optimole` cases["my exports"] = `"my exports"` + cases[`C:\My Exports\`] = `"C:\My Exports\\"` } else { cases["my exports"] = "'my exports'" cases["it's"] = `'it'\''s'` cases["$HOME"] = "'$HOME'" + cases[`exports\archive`] = `'exports\archive'` } for in, want := range cases { if got := shellQuote(in); got != want { From 9c335fcb2714cd4c465cec60a7f238bb3679aaf8 Mon Sep 17 00:00:00 2001 From: selul Date: Wed, 4 Nov 2026 15:49:25 +0200 Subject: [PATCH 6/9] export: quote @, % and = in resume paths; note the Windows shell limits Co-Authored-By: Claude Opus 5.5 --- internal/cli/export.go | 14 ++++++++------ internal/cli/export_test.go | 5 +++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/cli/export.go b/internal/cli/export.go index 162aa29..3dab5b4 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -262,18 +262,20 @@ func startExport(ctx context.Context, env Env, sess *session) (int64, bool, int) return 0, false, exportError(env, sess, name, firstCancel(ctx, err)) } -// shellQuote writes a path for a command to paste: as is when it needs no -// quotes, else in single quotes, or on Windows in double quotes (a path -// cannot contain them), where a backslash run before the closing quote is -// doubled so it does not escape the quote. A backslash needs quotes outside -// Windows. +// shellQuote writes a path for a command to paste: as is when it holds only +// characters no shell treats specially, else in single quotes, or on Windows +// in double quotes (a path cannot contain them), where a backslash run before +// the closing quote is doubled so it does not escape the quote. A backslash +// needs quotes outside Windows. No one Windows form suits both cmd.exe and +// PowerShell: both still expand "%" or "$" inside double quotes, which a +// folder name rarely holds. func shellQuote(s string) string { windows := runtime.GOOS == "windows" plain := s != "" && strings.IndexFunc(s, func(r rune) bool { if r == '\\' { return !windows } - return !(r < utf8.RuneSelf && (unicode.IsLetter(r) || unicode.IsDigit(r)) || strings.ContainsRune("._-/:+,@%=", r)) + return !(r < utf8.RuneSelf && (unicode.IsLetter(r) || unicode.IsDigit(r)) || strings.ContainsRune("._-/:+,", r)) }) < 0 switch { case plain: diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go index f275c1f..b7aeb15 100644 --- a/internal/cli/export_test.go +++ b/internal/cli/export_test.go @@ -437,6 +437,8 @@ func TestShellQuote(t *testing.T) { } if runtime.GOOS == "windows" { cases[`C:\Backups\Optimole`] = `C:\Backups\Optimole` + cases["@exports"] = `"@exports"` + cases[`C:\%X%`] = `"C:\%X%"` cases["my exports"] = `"my exports"` cases[`C:\My Exports\`] = `"C:\My Exports\\"` } else { @@ -444,6 +446,9 @@ func TestShellQuote(t *testing.T) { cases["it's"] = `'it'\''s'` cases["$HOME"] = "'$HOME'" cases[`exports\archive`] = `'exports\archive'` + cases["=sh"] = "'=sh'" + cases["@exports"] = "'@exports'" + cases["100%"] = "'100%'" } for in, want := range cases { if got := shellQuote(in); got != want { From 8990156314db171614a058ecbfbb710f027092e8 Mon Sep 17 00:00:00 2001 From: selul Date: Wed, 4 Nov 2026 19:46:37 +0200 Subject: [PATCH 7/9] export: no "continues on the server" hint for a finished export Stopping the download of an export that had already finished printed "Export N continues on the server", which is not true. Seen on staging: Ctrl-C while downloading the single part of a completed export. The resume command is printed alone now; a running or not yet seen export keeps the full hint. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- internal/cli/export.go | 10 +++++++--- internal/cli/export_test.go | 19 ++++++++++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 010e7b4..9aee3a6 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ downloaded. It holds no download URLs. The archives are not unpacked. listed an archive with a new size (it rewrote it), that archive is downloaded again. - Download URLs are valid for a day; when storage refuses one, the CLI asks the server for a fresh one. Download URLs never get the API key and are never printed, and redirects are refused. -- Ctrl-C and `--timeout` stop only the CLI. The export continues on the server, and the command prints how to +- Ctrl-C and `--timeout` stop only the CLI, never the export on the server, and the command prints how to resume (`optimole export download ID`). There is no way to cancel an export from the API. #### Output diff --git a/internal/cli/export.go b/internal/cli/export.go index 3dab5b4..70d01ae 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -51,8 +51,8 @@ const downloadFlagsUsage = ` -o, --output DIR Folder the export folder goes ` const downloadNotes = ` -Ctrl-C and --timeout stop only the CLI: the export continues on the server, -and "optimole export download ID" picks up where the run stopped. +Ctrl-C and --timeout stop only the CLI, never the export on the server, and +"optimole export download ID" picks up where the run stopped. Exit codes: 0 the export completed and every selected archive is downloaded, 1 the export failed or was cancelled, an archive failed, or the timeout passed, @@ -372,7 +372,11 @@ func downloadExport(ctx, runCtx context.Context, env Env, sess *session, rep *ou if df.output != "." { resume += " -o " + shellQuote(df.output) } - fmt.Fprintf(env.Stderr, "Export %d continues on the server. Resume with: %s\n", id, resume) + if out.Export != nil && api.IsFinalStatus(out.Export.Status) { + fmt.Fprintf(env.Stderr, "Resume with: %s\n", resume) + } else { + fmt.Fprintf(env.Stderr, "Export %d continues on the server. Resume with: %s\n", id, resume) + } } switch { case interrupted: diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go index b7aeb15..0ca7aa5 100644 --- a/internal/cli/export_test.go +++ b/internal/cli/export_test.go @@ -225,7 +225,7 @@ func TestExportDownloadStopsAndTellsHowToResume(t *testing.T) { if code := h.run("export", "download", "latest", "-o", out, "--timeout", "150ms"); code != ExitFailure { t.Fatalf("timeout: exit %d", code) } - resume := "Resume with: optimole export download 1 -o " + shellQuote(out) + resume := "Export 1 continues on the server. Resume with: optimole export download 1 -o " + shellQuote(out) if !strings.Contains(h.stderr.String(), "timed out after 150ms") || !strings.Contains(h.stderr.String(), resume) { t.Errorf("timeout stderr: %s", h.stderr.String()) } @@ -243,6 +243,23 @@ func TestExportDownloadStopsAndTellsHowToResume(t *testing.T) { } } +func TestExportDownloadOfFinishedExportStopsWithoutServerHint(t *testing.T) { + h := newHarness(t).withKey() + h.srv.AddExport("completed", testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{archive(t, "a")}}) + h.srv.DownloadDelay = time.Second + out := t.TempDir() + + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(200*time.Millisecond, cancel) + if code := h.runCtx(ctx, "export", "download", "1", "-o", out); code != ExitInterrupted { + t.Fatalf("exit %d", code) + } + stderr := h.stderr.String() + if !strings.Contains(stderr, "Resume with: optimole export download 1 -o "+shellQuote(out)) || strings.Contains(stderr, "continues on the server") { + t.Errorf("stderr: %s", stderr) + } +} + func TestExportUsageErrors(t *testing.T) { h := newHarness(t).withKey() for _, args := range [][]string{ From 4a6daeff040e003bd3e4f0a5eece1c200a91e91f Mon Sep 17 00:00:00 2001 From: selul Date: Wed, 4 Nov 2026 20:24:03 +0200 Subject: [PATCH 8/9] export tests: stop the finished-export download on its request, not a timer The test cancelled after 200 ms, which on a slow runner could land before the CLI had read the export and so print the running-export hint. It now cancels when the archive request arrives, and the fake server's download delay ends when the client cancels. Co-Authored-By: Claude Opus 5.5 --- internal/cli/export_test.go | 10 ++++++++-- internal/testserver/exports.go | 6 +++++- internal/testserver/testserver.go | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/cli/export_test.go b/internal/cli/export_test.go index 0ca7aa5..74435ca 100644 --- a/internal/cli/export_test.go +++ b/internal/cli/export_test.go @@ -246,11 +246,17 @@ func TestExportDownloadStopsAndTellsHowToResume(t *testing.T) { func TestExportDownloadOfFinishedExportStopsWithoutServerHint(t *testing.T) { h := newHarness(t).withKey() h.srv.AddExport("completed", testserver.ExportDomain{Domain: "example.com", Archives: [][]byte{archive(t, "a")}}) - h.srv.DownloadDelay = time.Second + h.srv.DownloadDelay = 10 * time.Second out := t.TempDir() + // Stop once the archive download starts: by then the CLI has seen the + // export finished. ctx, cancel := context.WithCancel(context.Background()) - time.AfterFunc(200*time.Millisecond, cancel) + h.srv.Hook = func(route testserver.Route, _ *http.Request) { + if route == testserver.RouteDownload { + cancel() + } + } if code := h.runCtx(ctx, "export", "download", "1", "-o", out); code != ExitInterrupted { t.Fatalf("exit %d", code) } diff --git a/internal/testserver/exports.go b/internal/testserver/exports.go index 5f48d1f..9b233a3 100644 --- a/internal/testserver/exports.go +++ b/internal/testserver/exports.go @@ -415,7 +415,11 @@ func (s *Server) handleDownload(w http.ResponseWriter, r *http.Request) { s.mu.Unlock() }() if s.DownloadDelay > 0 { - time.Sleep(s.DownloadDelay) + select { + case <-time.After(s.DownloadDelay): + case <-r.Context().Done(): + return + } } if r.Header.Get("Authorization") != "" { s.violate("archive download carried an Authorization header") diff --git a/internal/testserver/testserver.go b/internal/testserver/testserver.go index c035a48..9973499 100644 --- a/internal/testserver/testserver.go +++ b/internal/testserver/testserver.go @@ -126,7 +126,8 @@ type Server struct { CompleteDelay time.Duration // PutDelay slows down storage uploads. PutDelay time.Duration - // DownloadDelay slows down archive downloads. + // DownloadDelay slows down archive downloads; a request the client + // cancels stops waiting and gets no answer. DownloadDelay time.Duration // UploadExpiresIn sets the expires_at of new upload URLs (default 15 // minutes; negative values give URLs that are already expired). From 4e6fe82d12d2a3f503486e21a8dc888bbfc55ca8 Mon Sep 17 00:00:00 2001 From: selul Date: Wed, 4 Nov 2026 20:28:43 +0200 Subject: [PATCH 9/9] export tests: count parallel downloads without relying on timing TestRunLimitsParallelDownloads relied on a 30 ms download delay to keep three downloads overlapping, and on a busy Windows runner with -race only two overlapped. The first three downloads now wait in the server hook until all three have arrived, then 50 ms more, so a correct limit always shows 3 and a broken one shows 2 or 4. The fake's own download counter had no other user and is removed. Co-Authored-By: Claude Opus 5.5 --- internal/export/export_test.go | 40 +++++++++++++++++++++++++++---- internal/testserver/exports.go | 17 ------------- internal/testserver/testserver.go | 3 --- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/internal/export/export_test.go b/internal/export/export_test.go index 471e181..6385032 100644 --- a/internal/export/export_test.go +++ b/internal/export/export_test.go @@ -793,13 +793,45 @@ func TestRunLimitsParallelDownloads(t *testing.T) { archives = append(archives, zipOf(t, fmt.Sprintf("f%d", i))) } id := f.srv.AddExport(api.ExportCompleted, testserver.ExportDomain{Domain: "example.com", Archives: archives}) - f.srv.DownloadDelay = 30 * time.Millisecond - out := Run(context.Background(), Options{Client: f.client, ExportID: id, Dir: f.dir, Concurrency: 3, Sleep: immediate}) + + // The first three downloads wait until all three have arrived, then a + // little longer, so a fourth one would be seen if the limit let it start. + const limit = 3 + var mu sync.Mutex + arrived, waiting, most := 0, 0, 0 + all := make(chan struct{}) + f.srv.Hook = func(route testserver.Route, _ *http.Request) { + if route != testserver.RouteDownload { + return + } + mu.Lock() + arrived++ + first := arrived <= limit + if arrived == limit { + close(all) + } + waiting++ + most = max(most, waiting) + mu.Unlock() + if first { + select { + case <-all: + time.Sleep(50 * time.Millisecond) + case <-time.After(5 * time.Second): + } + } + mu.Lock() + waiting-- + mu.Unlock() + } + out := Run(context.Background(), Options{Client: f.client, ExportID: id, Dir: f.dir, Concurrency: limit, Sleep: immediate}) if out.Err != nil || out.Downloaded != 7 { t.Fatalf("outcome: %+v", out) } - if n := f.srv.MaxConcurrentDownloads(); n != 3 { - t.Errorf("at most %d downloads at once, want 3", n) + mu.Lock() + defer mu.Unlock() + if most != limit { + t.Errorf("at most %d downloads at once, want %d", most, limit) } } diff --git a/internal/testserver/exports.go b/internal/testserver/exports.go index 9b233a3..44505e4 100644 --- a/internal/testserver/exports.go +++ b/internal/testserver/exports.go @@ -102,14 +102,6 @@ func (s *Server) ExpireArchiveURLs() { s.urlGeneration++ } -// MaxConcurrentDownloads is the highest number of archive downloads seen at -// once. -func (s *Server) MaxConcurrentDownloads() int { - s.mu.Lock() - defer s.mu.Unlock() - return s.maxDownloads -} - // ExportIDs lists the stored exports, oldest first. func (s *Server) ExportIDs() []int64 { s.mu.Lock() @@ -405,15 +397,6 @@ func (s *Server) handleDownload(w http.ResponseWriter, r *http.Request) { status := 0 var drop func() defer func() { s.finishThenDrop(ev, status, drop) }() - s.mu.Lock() - s.inflightDownloads++ - s.maxDownloads = max(s.maxDownloads, s.inflightDownloads) - s.mu.Unlock() - defer func() { - s.mu.Lock() - s.inflightDownloads-- - s.mu.Unlock() - }() if s.DownloadDelay > 0 { select { case <-time.After(s.DownloadDelay): diff --git a/internal/testserver/testserver.go b/internal/testserver/testserver.go index 9973499..4e399a1 100644 --- a/internal/testserver/testserver.go +++ b/internal/testserver/testserver.go @@ -159,9 +159,6 @@ type Server struct { exportTemplate []ExportDomain exportsPaused bool urlGeneration int - - inflightDownloads int - maxDownloads int } // New starts a fake and closes it when the test ends. The test fails if a