diff --git a/.env.example b/.env.example index 2b1cf9f1..2bd6f7f4 100644 --- a/.env.example +++ b/.env.example @@ -55,14 +55,9 @@ DATABASE_PASSWORD=change-me # Feature flags # STACKABLE_COCKPIT_COMPLETION_ENABLED=false # Disable SQL editor code completion (default: true) # STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED=true # Enable S3/HDFS file browser (default: false) -# PUBLIC_STACKABLE_COCKPIT_STORAGE_RESTORE_TABS=true # File browser: continue where you left off — restore open tabs from the last session (default: false) -# PUBLIC_STACKABLE_COCKPIT_INFINITE_SCROLL_ENABLED=true # Load CSV and Parquet preview rows lazily while scrolling; any other value than "true" disables it (default: true) -# PUBLIC_STACKABLE_COCKPIT_STORAGE_CUT_COPY_ENABLED=true # Enables cut/copy and dragging in the storage browser; client-only, the server work is gated by paste/move (default: false) -# PUBLIC_STACKABLE_COCKPIT_STORAGE_PASTE_ENABLED=true # Enables paste in the storage browser; the transfer API accepts copies only with this, and moves with this or MOVE_ENABLED (default: false) -# PUBLIC_STACKABLE_COCKPIT_STORAGE_RENAME_ENABLED=true # Enables rename in the storage browser; also enforced by the transfer API (default: false) -# PUBLIC_STACKABLE_COCKPIT_STORAGE_MOVE_ENABLED=true # Enables drag-and-drop move in the storage browser (dragging also needs CUT_COPY_ENABLED); the transfer API accepts moves with this or PASTE_ENABLED (default: false) +# PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED=true # Enables cut, copy, paste, rename and drag-and-drop move in the storage browser; also enforced by the transfer API (default: false) -# Storage browser limits and settings +# Storage browser limits and settings. Invalid values stop the server at startup. # STACKABLE_COCKPIT_TEXT_PREVIEW_BYTES=262144 # Max bytes fetched for text, TSV and JSON previews; editable files are fetched completely (default: 262144) # STACKABLE_COCKPIT_IMAGE_PREVIEW_BYTES=5242880 # Max bytes fetched for image previews (default: 5242880) # STACKABLE_COCKPIT_PDF_PREVIEW_BYTES=26214400 # Max bytes fetched for PDF previews (default: 26214400) diff --git a/.env.test b/.env.test index 24624272..23c4b8ae 100644 --- a/.env.test +++ b/.env.test @@ -12,16 +12,11 @@ STACKABLE_COCKPIT_PDF_PREVIEW_BYTES=26214400 STACKABLE_COCKPIT_FILE_PREVIEW_ROWS=250 STACKABLE_COCKPIT_FILE_PREVIEW_COLUMNS=50 STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES=gzip-no_offset -PUBLIC_STACKABLE_COCKPIT_STORAGE_RESTORE_TABS=true PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES=25,50,100 PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE=25 PUBLIC_STACKABLE_COCKPIT_MAX_RECENT_FILES=15 PUBLIC_STACKABLE_COCKPIT_UPLOAD_CONCURRENCY=3 -PUBLIC_STACKABLE_COCKPIT_INFINITE_SCROLL_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_CUT_COPY_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_PASTE_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_RENAME_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_MOVE_ENABLED=true +PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED=true # One object at a time, so the cancel E2E test can stop a move mid-way. STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY=1 GARAGE_ADMIN_URL=http://localhost:30902 diff --git a/README.md b/README.md index 8cbda0ad..d11426e4 100644 --- a/README.md +++ b/README.md @@ -92,17 +92,15 @@ docker run -p 3000:3000 cockpit:0.0.0-dev The application is configured via environment variables. Create a `.env` file at the project root, or set them in your deployment environment. +Invalid values (e.g. `abc` for an integer) stop the server at startup with an error that names the invalid variables. ### Feature Flags -| Variable | Type | Default | Description | Example | -| --------------------------------------------------- | -------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | -| `STACKABLE_COCKPIT_COMPLETION_ENABLED` | `boolean` (`"false"` to disable) | `true` | Enables the SQL editor code-completion provider and the metadata endpoint. Set to `"false"` to fall back to plain syntax highlighting. | `STACKABLE_COCKPIT_COMPLETION_ENABLED=false` | -| `STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED` | `boolean` (`"true"` to enable) | `false` | Shows the S3/HDFS file browser in the sidebar and activates routes under `/storage`. Must be explicitly opted in to expose storage credentials and the file-browser UI. | `STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED=true` | -| `PUBLIC_STACKABLE_COCKPIT_STORAGE_CUT_COPY_ENABLED` | `boolean` (`"true"` to enable) | `false` | Shows Cut/Copy in the file browser and lets items be dragged. Client-only: it fills the clipboard, the S3 work happens on paste or move. | `PUBLIC_STACKABLE_COCKPIT_STORAGE_CUT_COPY_ENABLED=true` | -| `PUBLIC_STACKABLE_COCKPIT_STORAGE_PASTE_ENABLED` | `boolean` (`"true"` to enable) | `false` | Shows Paste in the file browser. When disabled, the server rejects copies via `/api/storage/transfer` with 404, and moves too unless the Move flag is enabled. | `PUBLIC_STACKABLE_COCKPIT_STORAGE_PASTE_ENABLED=true` | -| `PUBLIC_STACKABLE_COCKPIT_STORAGE_RENAME_ENABLED` | `boolean` (`"true"` to enable) | `false` | Shows Rename in the file browser. The server accepts renames via `/api/storage/transfer` only when this is enabled. | `PUBLIC_STACKABLE_COCKPIT_STORAGE_RENAME_ENABLED=true` | -| `PUBLIC_STACKABLE_COCKPIT_STORAGE_MOVE_ENABLED` | `boolean` (`"true"` to enable) | `false` | Enables drag-and-drop move in the file browser (dragging also needs cut/copy). The server accepts moves via `/api/storage/transfer` only when this or paste is enabled, since cut-and-paste also moves. | `PUBLIC_STACKABLE_COCKPIT_STORAGE_MOVE_ENABLED=true` | +| Variable | Type | Default | Description | Example | +| --------------------------------------------------- | -------------------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | +| `STACKABLE_COCKPIT_COMPLETION_ENABLED` | `boolean` (`"false"` to disable) | `true` | Enables the SQL editor code-completion provider and the metadata endpoint. Set to `"false"` to fall back to plain syntax highlighting. | `STACKABLE_COCKPIT_COMPLETION_ENABLED=false` | +| `STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED` | `boolean` (`"true"` to enable) | `false` | Shows the S3/HDFS file browser in the sidebar and activates routes under `/storage`. Must be explicitly opted in to expose storage credentials and the file-browser UI. | `STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED=true` | +| `PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED` | `boolean` (`"true"` to enable) | `false` | Shows Cut, Copy, Paste and Rename in the file browser and enables drag-and-drop move. When disabled, the server rejects copies, moves and renames via `/api/storage/transfer` with 404. | `PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED=true` | ### Storage Preview Limits @@ -115,9 +113,9 @@ Create a `.env` file at the project root, or set them in your deployment environ ### Storage Transfers -| Variable | Type | Default | Description | Example | -| ------------------------------------------------ | --------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | -| `STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY` | `integer` | `8` | Objects copied at once within one copy, move or rename request. Copies run server-side in the storage backend (S3 CopyObject); no data streams through the app. A missing or invalid value falls back to the default. | `STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY=4` | +| Variable | Type | Default | Description | Example | +| ------------------------------------------------ | --------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| `STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY` | `integer` | `8` | Objects copied at once within one copy, move or rename request. Copies run server-side in the storage backend (S3 CopyObject); no data streams through the app. | `STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY=4` | ### Request Body Size diff --git a/deploy/helm/cockpit/README.md b/deploy/helm/cockpit/README.md index c70aba91..245972ac 100644 --- a/deploy/helm/cockpit/README.md +++ b/deploy/helm/cockpit/README.md @@ -95,8 +95,7 @@ The following table lists the configurable parameters of the Cockpit chart and t | --- | --- | --- | | `config.nodeEnv` | Node.js environment | `production` | | `config.bodySizeLimit` | Sets `BODY_SIZE_LIMIT`, the maximum request body size (e.g. `12M`). Must be at least the editable file size limit (`PUBLIC_STACKABLE_COCKPIT_MAX_EDITABLE_FILE_SIZE`, 5 MiB by default), otherwise saving larger files in the text editor fails. | `""` (image default `6M`) | -| `config.archiveCacheMaxMb` | Sets `STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB`, the maximum total size in MB of the archive preview cache in `/tmp`. Keep it below `tmp.sizeLimit`. | `""` (app default `512`) | -| `tmp.sizeLimit` | Size limit of the `/tmp` emptyDir. The kubelet evicts the pod when it is exceeded. Keep it above `config.archiveCacheMaxMb`. Empty means no limit. | `1Gi` | +| `tmp.sizeLimit` | Size limit of the `/tmp` emptyDir. The kubelet evicts the pod when it is exceeded. Keep it above `storage.archiveCacheMaxMb`. Empty means no limit. | `1Gi` | ### Database Parameters @@ -122,26 +121,29 @@ All feature parameters are optional. Omit them to use the application defaults. | --- | --- | --- | | `features.trino.completion.enabled` | Enable SQL code completion. | `true` (application default) | | `features.storage.enabled` | Show the S3/HDFS file browser and enable routes under `/storage`. | `false` (application default) | -| `features.storage.pageSizes` | Comma-separated page size options. | `25,50,100` (application default) | -| `features.storage.defaultPageSize` | Default page size; must be included in `pageSizes`. | First configured page size (application default) | -| `features.storage.maxRecentFiles` | Number of recently visited storage locations retained locally. | `15` (application default) | -| `features.storage.uploadConcurrency` | Maximum parallel requests for file uploads. | `3` (application default) | -| `features.storage.textPreviewBytes` | Maximum bytes fetched for text, TSV, and JSON previews. Editable text files (text types except CSV/TSV, up to the editable file size limit) are always fetched completely. | `262144` (application default) | -| `features.storage.imagePreviewBytes` | Maximum bytes fetched for image previews. | `5242880` (application default) | -| `features.storage.pdfPreviewBytes` | Maximum bytes fetched for PDF previews. | `26214400` (application default) | -| `features.storage.filePreviewRows` | Maximum rows in tabular file previews. | `250` (application default) | -| `features.storage.filePreviewColumns` | Maximum columns in tabular file previews. | `50` (application default) | -| `features.storage.transferConcurrency` | Objects copied at once within one copy, move or rename request. | `8` (application default) | -| `features.storage.restoreTabs` | Restore open file browser tabs from the last session. | `false` (application default) | -| `features.storage.infiniteScroll` | Load CSV and Parquet preview rows lazily while scrolling. | `true` (application default) | -| `features.storage.cutCopy` | Show Cut/Copy and allow dragging items. Client-only: the server work happens on paste or move. | `false` (application default) | -| `features.storage.paste` | Show Paste. The transfer API accepts copies only when this is enabled, and moves when this or `move` is enabled. | `false` (application default) | -| `features.storage.rename` | Show Rename. The transfer API only accepts renames when this is enabled. | `false` (application default) | -| `features.storage.move` | Allow drag-and-drop move. Dragging also needs `cutCopy`. The transfer API accepts moves when this or `paste` is enabled. | `false` (application default) | -| `features.storage.maxEditableFileSize` | Maximum size in bytes of files that can be edited and saved in the text editor. `config.bodySizeLimit` must be at least this large. | `5242880` (application default) | -| `features.storage.archivePreviewMaxMb` | Maximum size in MB of an archive opened for preview, checked against its compressed size, the total size of the listed entries and the size of an extracted entry. The archive cache is raised to at least twice this value, so keep `tmp.sizeLimit` above that. | `100` (application default) | -| `features.storage.parquetPreviewDisallowedCompressionTypes` | Comma-separated Parquet compression codecs that are not previewed. A `-no_offset` suffix (e.g. `gzip-no_offset`) only blocks files without an offset index. | `gzip-no_offset` (application default) | -| `features.storage.downloadHistoryRetentionDays` | Days a user's download history is kept. Integer from 1 to 3650. | `30` (application default) | +| `features.storage.transfer` | Show Cut, Copy, Paste and Rename and allow drag-and-drop move. The transfer API rejects copies, moves and renames while this is disabled. | `false` (application default) | + +### Storage Browser Parameters + +Limits and tuning for the file browser. Empty values use the application defaults; invalid values stop the application at startup. + +| Parameter | Description | Default | +| --- | --- | --- | +| `storage.pageSizes` | Comma-separated page size options. | `""` (application default `25,50,100`) | +| `storage.defaultPageSize` | Default page size; must be included in `pageSizes`. | `""` (application default Smallest page size) | +| `storage.maxRecentFiles` | Number of recently visited storage locations retained locally. | `""` (application default `15`) | +| `storage.uploadConcurrency` | Maximum parallel requests for file uploads. | `""` (application default `3`) | +| `storage.transferConcurrency` | Objects copied at once within one copy, move or rename request. | `""` (application default `8`) | +| `storage.textPreviewBytes` | Maximum bytes fetched for text, TSV, and JSON previews. Editable text files (text types except CSV/TSV, up to the editable file size limit) are always fetched completely. | `""` (application default `262144`) | +| `storage.imagePreviewBytes` | Maximum bytes fetched for image previews. | `""` (application default `5242880`) | +| `storage.pdfPreviewBytes` | Maximum bytes fetched for PDF previews. | `""` (application default `26214400`) | +| `storage.filePreviewRows` | Maximum rows in tabular file previews. | `""` (application default `250`) | +| `storage.filePreviewColumns` | Maximum columns in tabular file previews. | `""` (application default `50`) | +| `storage.maxEditableFileSize` | Maximum size in bytes of files that can be edited and saved in the text editor. `config.bodySizeLimit` must be at least this large. | `""` (application default `5242880`) | +| `storage.archivePreviewMaxMb` | Maximum size in MB of an archive opened for preview, checked against its compressed size, the total size of the listed entries and the size of an extracted entry. The archive cache is raised to at least twice this value, so keep `tmp.sizeLimit` above that. | `""` (application default `100`) | +| `storage.archiveCacheMaxMb` | Maximum total size in MB of the archive preview cache in `/tmp`. Keep it below `tmp.sizeLimit`. | `""` (application default `512`) | +| `storage.parquetPreviewDisallowedCompressionTypes` | Comma-separated Parquet compression codecs that are not previewed. A `-no_offset` suffix (e.g. `gzip-no_offset`) only blocks files without an offset index. | `""` (application default `gzip-no_offset`) | +| `storage.downloadHistoryRetentionDays` | Days a user's download history is kept. Integer from 1 to 3650. | `""` (application default `30`) | ### Authentication Parameters @@ -198,6 +200,13 @@ A configured CA certificate (`caCert` or `secretClass`) replaces the system trus | `extraVolumes` | Additional pod volumes, e.g. a Secret holding a CA certificate for `trino.tls.caCert`, `database.tls.caCert` or `storage.tls.caCert`. | `[]` | | `extraVolumeMounts` | Additional volume mounts for the Cockpit container. | `[]` | +### Environment Parameters + +| Parameter | Description | Default | +| --- | --- | --- | +| `extraEnv` | Additional environment variables for the Cockpit container, e.g. settings the chart does not expose. Do not repeat a variable the chart already sets. | `[]` | +| `extraEnvFrom` | Additional `envFrom` sources (ConfigMaps or Secrets) for the Cockpit container. | `[]` | + ### Resource Parameters | Parameter | Description | Default | diff --git a/deploy/helm/cockpit/templates/_helpers.tpl b/deploy/helm/cockpit/templates/_helpers.tpl index 9cca35d9..bd06d378 100644 --- a/deploy/helm/cockpit/templates/_helpers.tpl +++ b/deploy/helm/cockpit/templates/_helpers.tpl @@ -122,3 +122,19 @@ Database port, validated: int64 turns "" or "abc" into 0, which would fail only {{- end }} {{- $port }} {{- end }} + +{{/* +A value as a quoted environment variable. A whole number arrives as float64 and would render +as 5.24288e+06. Anything else is passed through unchanged for the application to validate. +*/}} +{{- define "cockpit.envValue" -}} +{{- if and (kindIs "float64" .) (eq (floor .) .) }}{{ int64 . | quote }}{{ else }}{{ . | quote }}{{ end }} +{{- end }} + +{{/* +"true" when a value is set: not omitted, null or an empty string. Unlike `with`, 0 and false +count as set, so the application rejects them instead of the chart dropping them. +*/}} +{{- define "cockpit.isSet" -}} +{{- if and (not (kindIs "invalid" .)) (ne (toString .) "") }}true{{ end }} +{{- end }} diff --git a/deploy/helm/cockpit/templates/deployment.yaml b/deploy/helm/cockpit/templates/deployment.yaml index 420a5167..60ed19bf 100644 --- a/deploy/helm/cockpit/templates/deployment.yaml +++ b/deploy/helm/cockpit/templates/deployment.yaml @@ -66,10 +66,6 @@ spec: - name: BODY_SIZE_LIMIT value: {{ . | quote }} {{- end }} - {{- with .Values.config.archiveCacheMaxMb }} - - name: STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB - value: {{ . | quote }} - {{- end }} {{- if hasKey $completion "enabled" }} - name: STACKABLE_COCKPIT_COMPLETION_ENABLED value: {{ index $completion "enabled" | quote }} @@ -79,86 +75,70 @@ spec: - name: STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED value: {{ .enabled | quote }} {{- end }} - {{- with .pageSizes }} + {{- if hasKey . "transfer" }} + - name: PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED + value: {{ .transfer | quote }} + {{- end }} + {{- end }} + {{- if include "cockpit.isSet" .Values.storage.pageSizes }} - name: PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES - value: {{ . | quote }} + value: {{ include "cockpit.envValue" .Values.storage.pageSizes }} {{- end }} - {{- with .defaultPageSize }} + {{- if include "cockpit.isSet" .Values.storage.defaultPageSize }} - name: PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE - value: {{ . | int64 | quote }} + value: {{ include "cockpit.envValue" .Values.storage.defaultPageSize }} {{- end }} - {{- with .maxRecentFiles }} + {{- if include "cockpit.isSet" .Values.storage.maxRecentFiles }} - name: PUBLIC_STACKABLE_COCKPIT_MAX_RECENT_FILES - value: {{ . | int64 | quote }} + value: {{ include "cockpit.envValue" .Values.storage.maxRecentFiles }} {{- end }} - {{- with .uploadConcurrency }} + {{- if include "cockpit.isSet" .Values.storage.uploadConcurrency }} - name: PUBLIC_STACKABLE_COCKPIT_UPLOAD_CONCURRENCY - value: {{ . | int64 | quote }} + value: {{ include "cockpit.envValue" .Values.storage.uploadConcurrency }} {{- end }} - {{- with .textPreviewBytes }} + {{- if include "cockpit.isSet" .Values.storage.transferConcurrency }} + - name: STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY + value: {{ include "cockpit.envValue" .Values.storage.transferConcurrency }} + {{- end }} + {{- if include "cockpit.isSet" .Values.storage.textPreviewBytes }} - name: STACKABLE_COCKPIT_TEXT_PREVIEW_BYTES - value: {{ . | int64 | quote }} + value: {{ include "cockpit.envValue" .Values.storage.textPreviewBytes }} {{- end }} - {{- with .imagePreviewBytes }} + {{- if include "cockpit.isSet" .Values.storage.imagePreviewBytes }} - name: STACKABLE_COCKPIT_IMAGE_PREVIEW_BYTES - value: {{ . | int64 | quote }} + value: {{ include "cockpit.envValue" .Values.storage.imagePreviewBytes }} {{- end }} - {{- with .pdfPreviewBytes }} + {{- if include "cockpit.isSet" .Values.storage.pdfPreviewBytes }} - name: STACKABLE_COCKPIT_PDF_PREVIEW_BYTES - value: {{ . | int64 | quote }} + value: {{ include "cockpit.envValue" .Values.storage.pdfPreviewBytes }} {{- end }} - {{- with .filePreviewRows }} + {{- if include "cockpit.isSet" .Values.storage.filePreviewRows }} - name: STACKABLE_COCKPIT_FILE_PREVIEW_ROWS - value: {{ . | int64 | quote }} + value: {{ include "cockpit.envValue" .Values.storage.filePreviewRows }} {{- end }} - {{- with .filePreviewColumns }} + {{- if include "cockpit.isSet" .Values.storage.filePreviewColumns }} - name: STACKABLE_COCKPIT_FILE_PREVIEW_COLUMNS - value: {{ . | int64 | quote }} - {{- end }} - {{- with .transferConcurrency }} - - name: STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY - value: {{ . | int64 | quote }} - {{- end }} - {{- if hasKey . "restoreTabs" }} - - name: PUBLIC_STACKABLE_COCKPIT_STORAGE_RESTORE_TABS - value: {{ .restoreTabs | quote }} - {{- end }} - {{- if hasKey . "infiniteScroll" }} - - name: PUBLIC_STACKABLE_COCKPIT_INFINITE_SCROLL_ENABLED - value: {{ .infiniteScroll | quote }} + value: {{ include "cockpit.envValue" .Values.storage.filePreviewColumns }} {{- end }} - {{- if hasKey . "cutCopy" }} - - name: PUBLIC_STACKABLE_COCKPIT_STORAGE_CUT_COPY_ENABLED - value: {{ .cutCopy | quote }} - {{- end }} - {{- if hasKey . "paste" }} - - name: PUBLIC_STACKABLE_COCKPIT_STORAGE_PASTE_ENABLED - value: {{ .paste | quote }} - {{- end }} - {{- if hasKey . "rename" }} - - name: PUBLIC_STACKABLE_COCKPIT_STORAGE_RENAME_ENABLED - value: {{ .rename | quote }} - {{- end }} - {{- if hasKey . "move" }} - - name: PUBLIC_STACKABLE_COCKPIT_STORAGE_MOVE_ENABLED - value: {{ .move | quote }} - {{- end }} - {{- with .maxEditableFileSize }} + {{- if include "cockpit.isSet" .Values.storage.maxEditableFileSize }} - name: PUBLIC_STACKABLE_COCKPIT_MAX_EDITABLE_FILE_SIZE - value: {{ . | int64 | quote }} + value: {{ include "cockpit.envValue" .Values.storage.maxEditableFileSize }} {{- end }} - {{- with .archivePreviewMaxMb }} + {{- if include "cockpit.isSet" .Values.storage.archivePreviewMaxMb }} - name: STACKABLE_COCKPIT_ARCHIVE_PREVIEW_MAX_MB - value: {{ . | int64 | quote }} + value: {{ include "cockpit.envValue" .Values.storage.archivePreviewMaxMb }} + {{- end }} + {{- if include "cockpit.isSet" .Values.storage.archiveCacheMaxMb }} + - name: STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB + value: {{ include "cockpit.envValue" .Values.storage.archiveCacheMaxMb }} {{- end }} - {{- with .parquetPreviewDisallowedCompressionTypes }} + {{- if include "cockpit.isSet" .Values.storage.parquetPreviewDisallowedCompressionTypes }} - name: STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES - value: {{ . | quote }} + value: {{ include "cockpit.envValue" .Values.storage.parquetPreviewDisallowedCompressionTypes }} {{- end }} - {{- with .downloadHistoryRetentionDays }} + {{- if include "cockpit.isSet" .Values.storage.downloadHistoryRetentionDays }} - name: STACKABLE_COCKPIT_DOWNLOAD_HISTORY_RETENTION_DAYS - value: {{ . | int64 | quote }} - {{- end }} + value: {{ include "cockpit.envValue" .Values.storage.downloadHistoryRetentionDays }} {{- end }} {{- if .Values.trino.queryTtl }} - name: STACKABLE_COCKPIT_QUERY_TTL @@ -309,6 +289,13 @@ spec: - name: STACKABLE_COCKPIT_TRINO_TLS_CA_CERT value: /stackable/trino-tls/ca.crt {{- end }} + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} volumeMounts: - name: tmp mountPath: /tmp diff --git a/deploy/helm/cockpit/values.yaml b/deploy/helm/cockpit/values.yaml index c7d4e5c5..0e810288 100644 --- a/deploy/helm/cockpit/values.yaml +++ b/deploy/helm/cockpit/values.yaml @@ -102,7 +102,7 @@ startupProbe: terminationGracePeriodSeconds: 30 -# Optional application feature overrides. All defaults below are applied by the application when omitted. +# Optional feature toggles. All defaults below are applied by the application when omitted. # features: # trino: # completion: @@ -111,51 +111,9 @@ terminationGracePeriodSeconds: 30 # storage: # # Show the S3/HDFS file browser and enable routes under /storage. Default: false (application default). # enabled: false -# # Comma-separated page size options for paginated list views. Default: "25,50,100" (application default). -# pageSizes: "25,50,100" -# # Page size selected when no user preference is stored. Must be included in pageSizes. Default: first page size (application default). -# defaultPageSize: 25 -# # Number of recently visited storage locations retained in local storage. Default: 15 (application default). -# maxRecentFiles: 15 -# # Maximum parallel requests for file uploads. Default: 3 (application default). -# uploadConcurrency: 3 -# # Maximum bytes fetched for text, TSV, and JSON previews; editable text files are fetched completely. Default: 262144 (application default). -# textPreviewBytes: 262144 -# # Maximum bytes fetched for image previews. Default: 5242880 (application default). -# imagePreviewBytes: 5242880 -# # Maximum bytes fetched for PDF previews. Default: 26214400 (application default). -# pdfPreviewBytes: 26214400 -# # Maximum rows in tabular file previews. Default: 250 (application default). -# filePreviewRows: 250 -# # Maximum columns in tabular file previews. Default: 50 (application default). -# filePreviewColumns: 50 -# # Objects copied at once within one copy, move or rename request. Default: 8 (application default). -# transferConcurrency: 8 -# # Restore open file browser tabs from the last session. Default: false (application default). -# restoreTabs: false -# # Load CSV and Parquet preview rows lazily while scrolling. Default: true (application default). -# infiniteScroll: true -# # Show Cut/Copy and allow dragging items. Client-only: the server work happens on paste or move. Default: false (application default). -# cutCopy: false -# # Show Paste; the transfer API accepts copies only with this, and moves with this or move. Default: false (application default). -# paste: false -# # Show Rename; the transfer API accepts renames. Default: false (application default). -# rename: false -# # Allow drag-and-drop move (dragging also needs cutCopy); the transfer API accepts moves with this or paste. Default: false (application default). -# move: false -# # Maximum size in bytes of files that can be edited and saved in the text editor. -# # config.bodySizeLimit must be at least this large. Default: 5242880 (application default). -# maxEditableFileSize: 5242880 -# # Maximum size in MB of an archive opened for preview, checked against its compressed size, -# # the total size of the listed entries and the size of an extracted entry. The archive cache -# # (config.archiveCacheMaxMb) is raised to at least twice this value. Default: 100 (application default). -# archivePreviewMaxMb: 100 -# # Comma-separated Parquet compression codecs that are not previewed. A "-no_offset" suffix -# # only blocks files without an offset index. Default: "gzip-no_offset" (application default). -# parquetPreviewDisallowedCompressionTypes: "gzip-no_offset" -# # Days a user's download history is kept. Integer from 1 to 3650. -# # Default: 30 (application default). -# downloadHistoryRetentionDays: 30 +# # Show Cut, Copy, Paste and Rename and allow drag-and-drop move. The transfer API rejects +# # copies, moves and renames while disabled. Default: false (application default). +# transfer: false logging: level: "" @@ -171,6 +129,15 @@ extraVolumeMounts: [] # mountPath: /stackable/database-ca # readOnly: true +# Additional environment variables for the Cockpit container, e.g. settings the chart does not expose. +extraEnv: [] + # - name: STACKABLE_COCKPIT_FILE_PREVIEW_ROWS + # value: "500" +# Additional sources of environment variables (ConfigMaps or Secrets) for the Cockpit container. +extraEnvFrom: [] + # - secretRef: + # name: cockpit-extra-env + nodeSelector: {} tolerations: [] @@ -184,15 +151,11 @@ config: # Maximum request body size (BODY_SIZE_LIMIT), e.g. "12M". Empty uses the image default (6M). # Must be at least the editable file size limit of the storage text editor (5 MiB by default). bodySizeLimit: "" - # Maximum total size in MB of the archive preview cache in /tmp - # (STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB). Empty uses the application default (512). - # Keep it below tmp.sizeLimit. - archiveCacheMaxMb: "" # Scratch volume mounted at /tmp, used for the archive preview cache. tmp: # Size limit of the emptyDir. The kubelet evicts the pod when it is exceeded, - # so keep it above config.archiveCacheMaxMb (512 MB by default). Empty means no limit. + # so keep it above storage.archiveCacheMaxMb (512 MB by default). Empty means no limit. sizeLimit: 1Gi # Session secret for signing cookies. Chart auto-generates one if secretKeyRef.name is empty. @@ -226,8 +189,46 @@ database: # The Secret Operator mounts ca.crt into /stackable/database-tls. secretClass: "" -# Storage browser settings that are not feature flags. +# Storage browser settings that are not feature flags. Empty or omitted values use the application +# defaults; invalid values stop the application at startup. storage: + # Comma-separated page size options for paginated list views. Default: "25,50,100". + pageSizes: "" + # Page size selected when no user preference is stored. Must be included in pageSizes. + # Default: the smallest page size. + defaultPageSize: "" + # Number of recently visited storage locations retained in local storage. Default: 15. + maxRecentFiles: "" + # Maximum parallel requests for file uploads. Default: 3. + uploadConcurrency: "" + # Objects copied at once within one copy, move or rename request. Default: 8. + transferConcurrency: "" + # Maximum bytes fetched for text, TSV, and JSON previews; editable text files are fetched + # completely. Default: 262144. + textPreviewBytes: "" + # Maximum bytes fetched for image previews. Default: 5242880. + imagePreviewBytes: "" + # Maximum bytes fetched for PDF previews. Default: 26214400. + pdfPreviewBytes: "" + # Maximum rows in tabular file previews. Default: 250. + filePreviewRows: "" + # Maximum columns in tabular file previews. Default: 50. + filePreviewColumns: "" + # Maximum size in bytes of files that can be edited and saved in the text editor. + # config.bodySizeLimit must be at least this large. Default: 5242880. + maxEditableFileSize: "" + # Maximum size in MB of an archive opened for preview, checked against its compressed size, + # the total size of the listed entries and the size of an extracted entry. The archive cache + # (archiveCacheMaxMb) is raised to at least twice this value. Default: 100. + archivePreviewMaxMb: "" + # Maximum total size in MB of the archive preview cache in /tmp. Keep it below tmp.sizeLimit. + # Default: 512. + archiveCacheMaxMb: "" + # Comma-separated Parquet compression codecs that are not previewed. A "-no_offset" suffix + # only blocks files without an offset index. Default: "gzip-no_offset". + parquetPreviewDisallowedCompressionTypes: "" + # Days a user's download history is kept. Integer from 1 to 3650. Default: 30. + downloadHistoryRetentionDays: "" # Key that encrypts stored storage connection credentials: a 64-character hex string (32 bytes), # e.g. from `openssl rand -hex 32`. Required when features.storage.enabled is true. # Keep it stable: changing or losing it makes all stored credentials unreadable. diff --git a/dev/modules/env.sh b/dev/modules/env.sh index 108d3e9e..92ff9015 100644 --- a/dev/modules/env.sh +++ b/dev/modules/env.sh @@ -32,17 +32,12 @@ STACKABLE_COCKPIT_FILE_PREVIEW_ROWS=250 STACKABLE_COCKPIT_FILE_PREVIEW_COLUMNS=50 STACKABLE_COCKPIT_ARCHIVE_PREVIEW_MAX_MB=100 STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES=gzip-no_offset -PUBLIC_STACKABLE_COCKPIT_STORAGE_RESTORE_TABS=true PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES=25,50,100 PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE=25 PUBLIC_STACKABLE_COCKPIT_MAX_RECENT_FILES=15 PUBLIC_STACKABLE_COCKPIT_MAX_EDITABLE_FILE_SIZE=5242880 PUBLIC_STACKABLE_COCKPIT_UPLOAD_CONCURRENCY=3 -PUBLIC_STACKABLE_COCKPIT_INFINITE_SCROLL_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_CUT_COPY_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_PASTE_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_RENAME_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_MOVE_ENABLED=true +PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED=true DATABASE_HOST=${POSTGRESQL_HOST:-$NODE_IP} DATABASE_PORT=31432 DATABASE_NAME=cockpit @@ -63,17 +58,12 @@ STACKABLE_COCKPIT_IMAGE_PREVIEW_BYTES=5242880 STACKABLE_COCKPIT_PDF_PREVIEW_BYTES=26214400 STACKABLE_COCKPIT_FILE_PREVIEW_ROWS=250 STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES=gzip-no_offset -PUBLIC_STACKABLE_COCKPIT_STORAGE_RESTORE_TABS=true PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES=25,50,100 PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE=25 PUBLIC_STACKABLE_COCKPIT_MAX_RECENT_FILES=15 PUBLIC_STACKABLE_COCKPIT_MAX_EDITABLE_FILE_SIZE=5242880 PUBLIC_STACKABLE_COCKPIT_UPLOAD_CONCURRENCY=3 -PUBLIC_STACKABLE_COCKPIT_INFINITE_SCROLL_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_CUT_COPY_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_PASTE_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_RENAME_ENABLED=true -PUBLIC_STACKABLE_COCKPIT_STORAGE_MOVE_ENABLED=true +PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED=true DATABASE_HOST=${POSTGRESQL_HOST:-$NODE_IP} DATABASE_PORT=31432 DATABASE_NAME=cockpit diff --git a/eslint.config.js b/eslint.config.js index e9db4541..e8f3d942 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -177,8 +177,8 @@ export default defineConfig( { target: './src/lib/server', from: './src/lib/client', - // The server re-exports the PUBLIC_ flags it enforces, so each is parsed once. - except: ['./feature-flags.ts'], + // The server re-exports the PUBLIC_ flags and config it enforces, so each is parsed once. + except: ['./feature-flags.ts', './config.ts'], message: 'Server code must not import client-only utilities.' }, { diff --git a/messages/de.json b/messages/de.json index a08e2e08..5a4aac41 100644 --- a/messages/de.json +++ b/messages/de.json @@ -402,7 +402,6 @@ "storage_preview_csv_rows": "Erste {count} Zeilen werden angezeigt", "storage_preview_csv_rows_complete": "Alle {count} Zeilen werden angezeigt", "storage_preview_parquet_rows": "Erste {count} von {total} Zeilen werden angezeigt (Parquet)", - "storage_preview_infinite_scroll_disabled": "Dies ist das Ende der Vorschau. Laden Sie die vollständige Datei herunter oder bitten Sie den Administrator, das unendliche Scrollen zu aktivieren, um weitere Zeilen in dieser Vorschau zu laden.", "storage_preview_pdf_too_large": "PDF ist zu groß für die Inline-Vorschau.", "storage_preview_image_too_large": "Bild ist zu groß für die Anzeige.", "storage_preview_image_too_large_desc": "Die Datei überschreitet die maximale Vorschaugröße. Laden Sie die vollständige Datei herunter, um sie anzuzeigen.", diff --git a/messages/en.json b/messages/en.json index 8bd19461..20a1b7e6 100644 --- a/messages/en.json +++ b/messages/en.json @@ -402,7 +402,6 @@ "storage_preview_csv_rows": "Showing first {count} rows", "storage_preview_csv_rows_complete": "Showing all {count} rows", "storage_preview_parquet_rows": "Showing first {count} of {total} rows (parquet)", - "storage_preview_infinite_scroll_disabled": "This is the end of the preview. Download the full file or ask the administrator to enable infinite scrolling to load more rows in this preview.", "storage_preview_pdf_too_large": "PDF is too large to preview inline.", "storage_preview_image_too_large": "Image is too large to display.", "storage_preview_image_too_large_desc": "The file exceeds the maximum preview size. Download the full file to view it.", diff --git a/src/hooks.server.ts b/src/hooks.server.ts index 086772ae..01f39965 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -10,6 +10,9 @@ import { requestLogger, logger } from '$lib/server/logging'; import { getConnectionFromHeader } from '$lib/server/storage/connection.js'; import { STORAGE_CONNECTION_ID_HEADER } from '$lib/storage/connection-id-header.js'; import { storageBrowserEnabled } from '$lib/server/feature-flags.js'; +// Imported for its side effect: parses the storage and PUBLIC_ client config at +// startup, so invalid values stop the server instead of failing the first request. +import '$lib/server/storage/config.js'; import { storageEncryptionKey } from '$lib/server/storage/encryption-key.js'; import { runMigrations } from '$lib/server/migrate.js'; diff --git a/src/lib/client/config.spec.ts b/src/lib/client/config.spec.ts new file mode 100644 index 00000000..13bfc7a9 --- /dev/null +++ b/src/lib/client/config.spec.ts @@ -0,0 +1,41 @@ +import { describe, it, expect } from 'vitest'; +import { parseClientConfig } from './config.js'; + +describe('parseClientConfig', () => { + it('uses the defaults when nothing is set', () => { + expect(parseClientConfig({})).toEqual({ + allowedPageSizes: [25, 50, 100], + defaultPageSize: 25, + maxEditableFileSize: 5 * 1024 * 1024, + maxRecentFiles: 15, + uploadConcurrency: 3 + }); + }); + + it('deduplicates and sorts page sizes', () => { + const config = parseClientConfig({ PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES: '100, 10,25,10' }); + expect(config.allowedPageSizes).toEqual([10, 25, 100]); + expect(config.defaultPageSize).toBe(10); + }); + + it('rejects invalid page sizes instead of dropping them', () => { + expect(() => parseClientConfig({ PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES: '10,abc' })).toThrow( + /PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES/ + ); + }); + + it('requires the default page size to be one of the page sizes', () => { + expect( + parseClientConfig({ + PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES: '10,20', + PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE: '20' + }).defaultPageSize + ).toBe(20); + expect(() => + parseClientConfig({ + PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES: '10,20', + PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE: '30' + }) + ).toThrow(/PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE: must be one of/); + }); +}); diff --git a/src/lib/client/config.ts b/src/lib/client/config.ts new file mode 100644 index 00000000..8673610e --- /dev/null +++ b/src/lib/client/config.ts @@ -0,0 +1,75 @@ +// Client-accessible configuration: limits and defaults, not feature toggles. +// Read at runtime via SvelteKit's $env/dynamic/public, so all variables must be +// prefixed PUBLIC_. Keep this module free of browser-only code: the server +// loads it too and enforces `maxEditableFileSize`. + +import { z } from 'zod'; +import { env } from '$env/dynamic/public'; +import { envInteger, envList, envOptionalInteger, parseEnv } from '$lib/env-schema.js'; + +const schema = z + .object({ + PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES: envList(['25', '50', '100']).pipe( + z + .array(z.coerce.number().int().positive()) + .min(1) + .transform((sizes) => [...new Set(sizes)].sort((a, b) => a - b)) + ), + PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE: envOptionalInteger, + PUBLIC_STACKABLE_COCKPIT_MAX_EDITABLE_FILE_SIZE: envInteger(5 * 1024 * 1024), + PUBLIC_STACKABLE_COCKPIT_MAX_RECENT_FILES: envInteger(15), + PUBLIC_STACKABLE_COCKPIT_UPLOAD_CONCURRENCY: envInteger(3) + }) + .refine( + (config) => + config.PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE === undefined || + config.PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES.includes( + config.PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE + ), + { + path: ['PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE'], + message: 'must be one of PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES' + } + ); + +export function parseClientConfig(env: Record) { + const config = parseEnv(schema, env); + const allowedPageSizes: readonly number[] = config.PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES; + return { + allowedPageSizes, + defaultPageSize: config.PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE ?? allowedPageSizes[0], + maxEditableFileSize: config.PUBLIC_STACKABLE_COCKPIT_MAX_EDITABLE_FILE_SIZE, + maxRecentFiles: config.PUBLIC_STACKABLE_COCKPIT_MAX_RECENT_FILES, + uploadConcurrency: config.PUBLIC_STACKABLE_COCKPIT_UPLOAD_CONCURRENCY + }; +} + +const config = parseClientConfig(env); + +/** The page sizes available in paginated list views, deduplicated and sorted + * ascending. Controlled by `PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES` + * (comma-separated positive integers). Default: `25,50,100`. */ +export const allowedPageSizes = config.allowedPageSizes; + +/** The page size selected when no user preference is stored. Controlled by + * `PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE`; must be one of + * `allowedPageSizes`. Default: the smallest allowed page size. */ +export const defaultPageSize = config.defaultPageSize; + +/** Maximum file size (in bytes) that can be edited inline in the text editor. + * Larger files show a read-only preview without the save button. Files up to + * this size are previewed completely (not truncated to the text preview + * limit) so they can be edited. Also enforced by the save endpoint, which + * rejects larger saves with HTTP 413. + * Controlled by `PUBLIC_STACKABLE_COCKPIT_MAX_EDITABLE_FILE_SIZE`. Default: 5242880 (5 MiB). */ +export const maxEditableFileSize = config.maxEditableFileSize; + +/** Maximum number of recently visited files and locations kept in localStorage + * for the storage browser history. + * Controlled by `PUBLIC_STACKABLE_COCKPIT_MAX_RECENT_FILES`. Default: 15. */ +export const maxRecentFiles = config.maxRecentFiles; + +/** Maximum number of parallel HTTP requests fired during the upload conflict + * check and during the upload itself. + * Controlled by `PUBLIC_STACKABLE_COCKPIT_UPLOAD_CONCURRENCY`. Default: 3. */ +export const uploadConcurrency = config.uploadConcurrency; diff --git a/src/lib/client/feature-flags.spec.ts b/src/lib/client/feature-flags.spec.ts deleted file mode 100644 index 1411143a..00000000 --- a/src/lib/client/feature-flags.spec.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { parsePageSizes } from './feature-flags.js'; - -describe('parsePageSizes', () => { - it('defaults to [25, 50, 100] when raw is undefined', () => { - expect(parsePageSizes(undefined)).toEqual([25, 50, 100]); - }); - - it('defaults to [25, 50, 100] when raw is empty string', () => { - expect(parsePageSizes('')).toEqual([25, 50, 100]); - }); - - it('parses comma-separated values', () => { - expect(parsePageSizes('10,20,50')).toEqual([10, 20, 50]); - }); - - it('deduplicates values', () => { - expect(parsePageSizes('25,50,25,100,50')).toEqual([25, 50, 100]); - }); - - it('sorts ascending', () => { - expect(parsePageSizes('100,25,50')).toEqual([25, 50, 100]); - }); - - it('filters out non-positive integers', () => { - expect(parsePageSizes('0,-5,10,abc,25')).toEqual([10, 25]); - }); - - it('trims whitespace around values', () => { - expect(parsePageSizes(' 10 , 25 , 50 ')).toEqual([10, 25, 50]); - }); - - it('falls back when all values are filtered out', () => { - expect(parsePageSizes('0,-1,abc')).toEqual([25, 50, 100]); - }); - - it('handles single value', () => { - expect(parsePageSizes('42')).toEqual([42]); - }); -}); diff --git a/src/lib/client/feature-flags.ts b/src/lib/client/feature-flags.ts index 2b39af62..5ae3a2e5 100644 --- a/src/lib/client/feature-flags.ts +++ b/src/lib/client/feature-flags.ts @@ -4,129 +4,18 @@ // here (see `$lib/server/feature-flags`), so each is parsed in one place. // Keep this module free of browser-only code: it is also loaded on the server. +import { z } from 'zod'; import { env } from '$env/dynamic/public'; +import { envBoolean, parseEnv } from '$lib/env-schema.js'; -/** When `PUBLIC_STACKABLE_COCKPIT_STORAGE_RESTORE_TABS=true`, the file browser - * saves open tabs (their name, order, and location) to localStorage and - * restores them the next time the user navigates to `/storage`. - * Disabled by default. */ -export const storageRestoreTabsEnabled = - (env.PUBLIC_STACKABLE_COCKPIT_STORAGE_RESTORE_TABS ?? - env.PUBLIC_STACKABLE_UI_STORAGE_RESTORE_TABS ?? - 'false') === 'true'; - -// ── Infinite scroll preview ─────────────────────────────────────────────────── - -/** When `PUBLIC_STACKABLE_COCKPIT_INFINITE_SCROLL_ENABLED=false`, the CSV and - * Parquet file previews load a fixed number of rows without virtual scrolling / - * infinite loading. Enabled by default — the previews fetch row chunks lazily - * as the user scrolls, reducing S3 costs and browser memory for large files. - * Any value other than `true` disables it. Also read by the server. - * Controlled by `PUBLIC_STACKABLE_COCKPIT_INFINITE_SCROLL_ENABLED`. Default: `true`. */ -export const infiniteScrollEnabled = - (env.PUBLIC_STACKABLE_COCKPIT_INFINITE_SCROLL_ENABLED ?? 'true') === 'true'; - -// ── Pagination ─────────────────────────────────────────────────────────────── - -/** Parse a comma-separated page-sizes string into a deduplicated, sorted list - * of positive integers. Falls back to [25, 50, 100] when the value is absent - * or produces no valid entries. */ -export function parsePageSizes(raw: string | undefined): readonly number[] { - if (raw) { - const parsed = raw - .split(',') - .map((s) => parseInt(s.trim(), 10)) - .filter((n) => Number.isFinite(n) && n > 0); - if (parsed.length > 0) { - return [...new Set(parsed)].sort((a, b) => a - b); - } - } - return [25, 50, 100]; -} - -/** The set of page sizes available in paginated list views. - * Controlled by `PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES` (comma-separated positive - * integers, e.g. `"10,25,50,100"`). Default: `25,50,100`. - * Values are deduplicated and sorted ascending. */ -export const allowedPageSizes: readonly number[] = parsePageSizes( - env.PUBLIC_STACKABLE_COCKPIT_PAGE_SIZES +const flags = parseEnv( + z.object({ PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED: envBoolean(false) }), + env ); -/** The page size selected by default when no user preference is stored. - * Controlled by `PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE`. - * Must be one of the values in `allowedPageSizes`; if the configured value - * is not in the allowed set it falls back to the first allowed size. */ -export const defaultPageSize: number = (() => { - const raw = env.PUBLIC_STACKABLE_COCKPIT_DEFAULT_PAGE_SIZE; - const parsed = parseInt(raw ?? '', 10); - return allowedPageSizes.includes(parsed) ? parsed : (allowedPageSizes[0] ?? 25); -})(); - -// ── Storage browser: Text editor ───────────────────────────────────────────── - -/** Maximum file size (in bytes) that can be edited inline in the text editor. - * Files larger than this will show a read-only preview without the save button. - * Controlled by `PUBLIC_STACKABLE_COCKPIT_MAX_EDITABLE_FILE_SIZE`. Default: 5242880 (5 MiB). - * Files up to this size are previewed completely (not truncated to the text - * preview limit) so they can be edited. Raise to allow editing larger files; - * lower to reduce preview bandwidth and browser memory. Also enforced by the - * save endpoint, which rejects larger saves with HTTP 413. */ -export const maxEditableFileSize: number = (() => { - const parsed = parseInt(env.PUBLIC_STACKABLE_COCKPIT_MAX_EDITABLE_FILE_SIZE ?? '', 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : 5 * 1024 * 1024; -})(); - -// ── Storage browser ────────────────────────────────────────────────────────── - -/** Maximum number of recently visited files and locations kept in localStorage - * for the storage browser history. - * Controlled by `PUBLIC_STACKABLE_COCKPIT_MAX_RECENT_FILES`. Default: 15. - * Raise to retain a longer history; lower to reduce localStorage pressure on - * deployments that handle many distinct objects. Must be a positive integer. */ -export const maxRecentFiles: number = (() => { - const parsed = parseInt(env.PUBLIC_STACKABLE_COCKPIT_MAX_RECENT_FILES ?? '', 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : 15; -})(); - -/** Maximum number of parallel HTTP requests fired during the upload conflict - * check and during the upload itself. - * Controlled by `PUBLIC_STACKABLE_COCKPIT_UPLOAD_CONCURRENCY`. Default: 3. - * Raise for faster bulk uploads on high-throughput connections; lower to - * reduce server pressure on constrained deployments. Must be a positive - * integer. */ -export const uploadConcurrency: number = (() => { - const parsed = parseInt(env.PUBLIC_STACKABLE_COCKPIT_UPLOAD_CONCURRENCY ?? '', 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : 3; -})(); - -// ── Storage browser: Context actions (cut, copy, paste, rename) ──────────── - -/** When `PUBLIC_STACKABLE_COCKPIT_STORAGE_CUT_COPY_ENABLED=true`, the context - * menu shows "Cut" and "Copy" actions, Ctrl+X / Ctrl+C keyboard shortcuts - * are active, and items can be dragged. Cut and copy only fill the clipboard; - * the costly S3 work (list + copy) happens on paste or move, which the server - * gates with those flags. Disabled by default. */ -export const storageCutCopyEnabled = - (env.PUBLIC_STACKABLE_COCKPIT_STORAGE_CUT_COPY_ENABLED ?? 'false') === 'true'; - -/** When `PUBLIC_STACKABLE_COCKPIT_STORAGE_PASTE_ENABLED=true`, the context - * menu shows "Paste" (when clipboard is non-empty) and Ctrl+V is active - * to paste cut/copied items into the current prefix. Also enforced by the - * transfer endpoint for copies and moves. Disabled by default. */ -export const storagePasteEnabled = - (env.PUBLIC_STACKABLE_COCKPIT_STORAGE_PASTE_ENABLED ?? 'false') === 'true'; - -/** When `PUBLIC_STACKABLE_COCKPIT_STORAGE_RENAME_ENABLED=true`, the context - * menu shows "Rename" and F2 is active to rename the selected item. Also - * enforced by the transfer endpoint. Disabled by default. */ -export const storageRenameEnabled = - (env.PUBLIC_STACKABLE_COCKPIT_STORAGE_RENAME_ENABLED ?? 'false') === 'true'; - -/** When `PUBLIC_STACKABLE_COCKPIT_STORAGE_MOVE_ENABLED=true`, drag-and-drop - * move is active (folder-row, breadcrumb, sidebar, and tab drop targets). - * This is a separate flag from paste because move is a costly S3 operation - * (list + copy + delete per directory level) and may be undesirable in - * high-egress-cost environments. Also enforced by the transfer endpoint. - * Disabled by default. */ -export const storageMoveEnabled = - (env.PUBLIC_STACKABLE_COCKPIT_STORAGE_MOVE_ENABLED ?? 'false') === 'true'; +/** When `PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED=true`, the file + * browser offers Cut, Copy, Paste and Rename (context menu and keyboard + * shortcuts), and items can be dragged to move them. Also enforced by the + * transfer endpoint as cost control: copies and moves list and copy (and, for + * moves, delete) every object in the storage backend. Disabled by default. */ +export const storageTransferEnabled = flags.PUBLIC_STACKABLE_COCKPIT_STORAGE_TRANSFER_ENABLED; diff --git a/src/lib/components/Pagination.svelte b/src/lib/components/Pagination.svelte index 280d36c1..d9cdfdcf 100644 --- a/src/lib/components/Pagination.svelte +++ b/src/lib/components/Pagination.svelte @@ -6,7 +6,7 @@ import * as m from '$lib/paraglide/messages.js'; import { browser } from '$app/environment'; import { isPageSize, type PageSize } from '$lib/types/pagination.js'; - import { allowedPageSizes } from '$lib/client/feature-flags.js'; + import { allowedPageSizes } from '$lib/client/config.js'; interface Props { /** Bound page size — parent initialises this from `initPageSize()`. */ diff --git a/src/lib/components/storage/explorer/FileExplorer.svelte b/src/lib/components/storage/explorer/FileExplorer.svelte index a62eefdc..fdd17a84 100644 --- a/src/lib/components/storage/explorer/FileExplorer.svelte +++ b/src/lib/components/storage/explorer/FileExplorer.svelte @@ -6,11 +6,7 @@ import { getStorageState } from '$lib/storage/context.js'; import { getTabsState } from '$lib/storage/context.js'; import { collectDroppedFiles } from '$lib/storage/file-collection.js'; - import { - storageCutCopyEnabled, - storagePasteEnabled, - storageRenameEnabled - } from '$lib/client/feature-flags.js'; + import { storageTransferEnabled } from '$lib/client/feature-flags.js'; import type { ContextMenuAction } from '$lib/storage/types.js'; import type { ActionName } from '$lib/storage/types.js'; import IconVisibility from 'virtual:icons/material-symbols/visibility'; @@ -227,7 +223,7 @@ icon: IconPaste, label: m.storage_action_paste(), disabled: !storage.clipboardState.canPaste || storage.archive.isInArchive, - hidden: !storagePasteEnabled || storage.archive.isInArchive + hidden: !storageTransferEnabled || storage.archive.isInArchive } ]; } @@ -259,28 +255,28 @@ icon: IconCut, label: m.storage_action_cut(), disabled: selectionCount === 0 || storage.archive.isInArchive, - hidden: !storageCutCopyEnabled || storage.archive.isInArchive + hidden: !storageTransferEnabled || storage.archive.isInArchive }, { key: 'copy' as ActionName, icon: IconCopy, label: m.storage_action_copy(), disabled: selectionCount === 0 || storage.archive.isInArchive, - hidden: !storageCutCopyEnabled || storage.archive.isInArchive + hidden: !storageTransferEnabled || storage.archive.isInArchive }, { key: 'paste' as ActionName, icon: IconPaste, label: m.storage_action_paste(), disabled: !storage.clipboardState.canPaste || storage.archive.isInArchive, - hidden: !storagePasteEnabled || storage.archive.isInArchive + hidden: !storageTransferEnabled || storage.archive.isInArchive }, { key: 'rename' as ActionName, icon: IconDriveFileRenameOutline, label: m.storage_action_rename(), disabled: selectionCount !== 1 || storage.archive.isInArchive, - hidden: !storageRenameEnabled || storage.archive.isInArchive + hidden: !storageTransferEnabled || storage.archive.isInArchive }, { key: 'copy-filename' as ActionName, diff --git a/src/lib/components/storage/explorer/FileRow.svelte b/src/lib/components/storage/explorer/FileRow.svelte index 519f013e..f690283e 100644 --- a/src/lib/components/storage/explorer/FileRow.svelte +++ b/src/lib/components/storage/explorer/FileRow.svelte @@ -5,7 +5,7 @@ import TimestampDisplay from '$lib/components/storage/shared/TimestampDisplay.svelte'; import FileIconAndName from './file-icon/FileIconAndName.svelte'; import { getStorageState } from '$lib/storage/context.js'; - import { storageCutCopyEnabled } from '$lib/client/feature-flags.js'; + import { storageTransferEnabled } from '$lib/client/feature-flags.js'; import { handleRowDragStart } from '$lib/storage/drag-handlers.js'; import * as m from '$lib/paraglide/messages.js'; @@ -36,7 +36,7 @@ : selected ? 'bg-primary/10 hover:bg-primary/15' : 'hover:bg-base-200/60'}" - draggable={storageCutCopyEnabled && !storage.archive.isInArchive} + draggable={storageTransferEnabled && !storage.archive.isInArchive} ondragstart={handleDragStart} onclick={(e) => storage.toggleSelect(file.key, e.ctrlKey || e.metaKey, e.shiftKey)} ondblclick={() => { diff --git a/src/lib/components/storage/explorer/FolderRow.svelte b/src/lib/components/storage/explorer/FolderRow.svelte index fa7a88bd..a565f0e4 100644 --- a/src/lib/components/storage/explorer/FolderRow.svelte +++ b/src/lib/components/storage/explorer/FolderRow.svelte @@ -4,7 +4,7 @@ import { keyToName } from '$lib/storage/utils.js'; import type { StorageObject } from '$lib/storage/types.js'; import { getStorageState } from '$lib/storage/context.js'; - import { storageCutCopyEnabled } from '$lib/client/feature-flags.js'; + import { storageTransferEnabled } from '$lib/client/feature-flags.js'; import { handleRowDragStart, parseStorageDrop, @@ -80,7 +80,7 @@ : selected ? 'bg-primary/10 hover:bg-primary/15' : 'hover:bg-base-200/60'}" - draggable={storageCutCopyEnabled && !storage.archive.isInArchive} + draggable={storageTransferEnabled && !storage.archive.isInArchive} ondragstart={handleDragStart} ondragover={handleDragOver} ondragleave={handleDragLeave} diff --git a/src/lib/components/storage/explorer/StorageBreadcrumb.svelte b/src/lib/components/storage/explorer/StorageBreadcrumb.svelte index c28f2616..dc42ba69 100644 --- a/src/lib/components/storage/explorer/StorageBreadcrumb.svelte +++ b/src/lib/components/storage/explorer/StorageBreadcrumb.svelte @@ -19,7 +19,7 @@ import type { StorageLocation } from '$lib/storage/types.js'; import { keyToName } from '$lib/storage/utils.js'; import { objectNameError } from '$lib/storage/name-validation.js'; - import { storageMoveEnabled } from '$lib/client/feature-flags.js'; + import { storageTransferEnabled } from '$lib/client/feature-flags.js'; import { parseStorageDrop, canStorageDrop } from '$lib/storage/drag-handlers.js'; import OperationsButton from './OperationsButton.svelte'; import ContextMenu from './ContextMenu.svelte'; @@ -345,7 +345,7 @@ class="dropdown" class:dropdown-open={collapsedDropdownOpen} ondragover={(e) => { - if (!storageMoveEnabled) return; + if (!storageTransferEnabled) return; e.preventDefault(); }} ondragenter={() => { diff --git a/src/lib/components/storage/explorer/TabBar.svelte b/src/lib/components/storage/explorer/TabBar.svelte index ad5f0572..c5c608d9 100644 --- a/src/lib/components/storage/explorer/TabBar.svelte +++ b/src/lib/components/storage/explorer/TabBar.svelte @@ -8,7 +8,7 @@ import type { TabsState } from '$lib/storage/tabs.svelte.js'; import type { ContextMenuAction } from '$lib/storage/types.js'; import { getStorageState } from '$lib/storage/context.js'; - import { storageMoveEnabled } from '$lib/client/feature-flags.js'; + import { storageTransferEnabled } from '$lib/client/feature-flags.js'; import { parseStorageDrop } from '$lib/storage/drag-handlers.js'; import ContextMenu from './ContextMenu.svelte'; import TooltipTrigger from '$lib/components/TooltipTrigger.svelte'; @@ -72,7 +72,8 @@ * not a tab-reorder drag (which uses text/plain). */ function isFileDrag(e: DragEvent): boolean { return ( - storageMoveEnabled && (e.dataTransfer?.types.includes('application/x-storage-keys') ?? false) + storageTransferEnabled && + (e.dataTransfer?.types.includes('application/x-storage-keys') ?? false) ); } diff --git a/src/lib/components/storage/explorer/TabBar.svelte.spec.ts b/src/lib/components/storage/explorer/TabBar.svelte.spec.ts index 7324aad1..5db468bb 100644 --- a/src/lib/components/storage/explorer/TabBar.svelte.spec.ts +++ b/src/lib/components/storage/explorer/TabBar.svelte.spec.ts @@ -16,7 +16,7 @@ function makeStorage(bucket = 'test-bucket', prefix = ''): StorageState { } function makeTabsState(storage: StorageState, tabCount = 2): TabsState { - const ts = new TabsState(storage, { persistEnabled: false }); + const ts = new TabsState(storage); ts.ensureInitialTab(); for (let i = 1; i < tabCount; i++) { ts.addTab(); diff --git a/src/lib/components/storage/modals/PreviewModal.svelte b/src/lib/components/storage/modals/PreviewModal.svelte index 1aef7bab..fddc343b 100644 --- a/src/lib/components/storage/modals/PreviewModal.svelte +++ b/src/lib/components/storage/modals/PreviewModal.svelte @@ -24,7 +24,7 @@ import { StorageError } from '$lib/storage/errors.js'; import { decodePreviewText } from '$lib/storage/text-decode.js'; import { addToast } from '$lib/stores/toast.svelte.js'; - import { maxEditableFileSize, infiniteScrollEnabled } from '$lib/client/feature-flags.js'; + import { maxEditableFileSize } from '$lib/client/config.js'; import { getStorageState } from '$lib/storage/context.js'; interface Props { @@ -136,8 +136,7 @@ let parquetShowingRowsCount = $state(0), csvShowingRowsCount = $state(0); let csvTotalRows = $state(0); - let parquetTab: 'metadata' | 'data' = $state('metadata'), - parquetDataLoading = $state(false); + let parquetTab: 'metadata' | 'data' = $state('metadata'); let editorText = $state(''); let originalText = $state(''); let saving = $state(false); @@ -195,7 +194,6 @@ imageNaturalWidth = 0; imageNaturalHeight = 0; parquetTab = 'metadata'; - parquetDataLoading = false; parquetShowingRowsCount = 0; csvShowingRowsCount = 0; csvTotalRows = 0; @@ -346,26 +344,14 @@ } = await readCsvNdjsonStream(res); csvTotalRows = initialTotal; - if (csvRows.length > 0 || infiniteScrollEnabled || !objectKey) { - preview = { - kind: 'csv_scroll', - headers: csvHeaders, - rows: csvRows, - totalRows: initialTotal, - truncated: initialTotal > csvRows.length, - totalSize - }; - } else { - const data = await fetchCsvRows(0, 250).catch(() => []); - preview = { - kind: 'csv_scroll', - headers: csvHeaders, - rows: data, - totalRows: csvTotalRows, - truncated: csvTotalRows > data.length, - totalSize - }; - } + preview = { + kind: 'csv_scroll', + headers: csvHeaders, + rows: csvRows, + totalRows: initialTotal, + truncated: initialTotal > csvRows.length, + totalSize + }; return; } @@ -675,57 +661,6 @@ } }); - // When infinite scroll is disabled and user clicks the data tab, - // fetch the first page of parquet data on demand. - $effect(() => { - if ( - preview.kind === 'parquet' && - parquetTab === 'data' && - preview.rows.length === 0 && - !infiniteScrollEnabled && - !parquetDataLoading && - objectKey - ) { - parquetDataLoading = true; - const hdrs = preview.headers; - const rowCount = Math.min(250, preview.totalRows); - const rows: unknown[][] = Array.from({ length: rowCount }, () => - new Array(hdrs.length).fill(undefined) - ); - preview = { ...preview, rows: rows.map((r) => [...r]) }; - const colPos: Record = {}; - fetchParquetRows(0, rowCount, (name, values) => { - const colIdx = hdrs.indexOf(name); - if (colIdx < 0) return; - let pos = colPos[name] ?? 0; - for (let i = 0; i < values.length; i++) { - while (rows.length <= pos) { - rows.push(new Array(hdrs.length).fill(undefined)); - } - rows[pos][colIdx] = values[i]; - pos++; - } - colPos[name] = pos; - if (preview.kind === 'parquet') { - preview = { ...preview, rows: rows.map((r) => [...r]) }; - } - }) - .then((finalRows) => { - parquetDataLoading = false; - if (preview.kind === 'parquet') { - preview = { - ...preview, - rows: finalRows, - truncated: finalRows.length < preview.totalRows - }; - } - }) - .catch(() => { - parquetDataLoading = false; - }); - } - }); - function toggleMaximized() { maximized = !maximized; } @@ -969,7 +904,7 @@ headers={preview.headers} initialRows={preview.rows} totalRows={preview.totalRows} - fetchRows={infiniteScrollEnabled ? fetchParquetRows : undefined} + fetchRows={fetchParquetRows} bind:showingRowsCount={parquetShowingRowsCount} hidden={parquetTab !== 'data'} /> @@ -1015,7 +950,7 @@ headers={preview.headers} initialRows={preview.rows} totalRows={csvTotalRows} - fetchRows={infiniteScrollEnabled ? fetchCsvRows : undefined} + fetchRows={fetchCsvRows} bind:showingRowsCount={csvShowingRowsCount} /> {:else if preview.kind === 'image'} diff --git a/src/lib/components/storage/modals/preview/CsvPreview.svelte b/src/lib/components/storage/modals/preview/CsvPreview.svelte index 7fdb2401..e59531a4 100644 --- a/src/lib/components/storage/modals/preview/CsvPreview.svelte +++ b/src/lib/components/storage/modals/preview/CsvPreview.svelte @@ -28,7 +28,6 @@ }: Props = $props(); const isLegacyMode = $derived(text !== '' && headers.length === 0); - const isSimpleMode = $derived(!isLegacyMode && !fetchRows && headers.length > 0); // ── Text mode: CSV/TSV content returned as text rather than structured rows ── @@ -124,39 +123,37 @@ } let totalTableWidth = $derived( - isLegacyMode || isSimpleMode ? 0 : LINE_NUM_WIDTH + sumColWidths(0, headers.length) + isLegacyMode ? 0 : LINE_NUM_WIDTH + sumColWidths(0, headers.length) ); let colStartIndex = $derived.by(() => { - if (isLegacyMode || isSimpleMode) return 0; + if (isLegacyMode) return 0; const idx = findDataColIndex(scrollLeft); return Math.max(0, Math.min(idx - COL_BUFFER, headers.length - 1)); }); let colEndIndex = $derived.by(() => { - if (isLegacyMode || isSimpleMode) return headers.length; + if (isLegacyMode) return headers.length; const idx = findDataColIndex(scrollLeft + containerWidth); return Math.min(headers.length, idx + 1 + COL_BUFFER); }); - let leftPadWidth = $derived(isLegacyMode || isSimpleMode ? 0 : sumColWidths(0, colStartIndex)); + let leftPadWidth = $derived(isLegacyMode ? 0 : sumColWidths(0, colStartIndex)); - let rightPadWidth = $derived( - isLegacyMode || isSimpleMode ? 0 : sumColWidths(colEndIndex, headers.length) - ); + let rightPadWidth = $derived(isLegacyMode ? 0 : sumColWidths(colEndIndex, headers.length)); let totalCellCount = $derived( - isLegacyMode || isSimpleMode + isLegacyMode ? headers.length + 1 : 1 + (colEndIndex - colStartIndex) + (leftPadWidth > 0 ? 1 : 0) + (rightPadWidth > 0 ? 1 : 0) ); onMount(() => { - if (!isLegacyMode && !isSimpleMode) populateChunk0(); + if (!isLegacyMode) populateChunk0(); }); $effect(() => { - if (isLegacyMode || isSimpleMode) return; + if (isLegacyMode) return; if (initialRows && initialRows.length > 0 && !hidden) { untrack(() => { populateChunk0(); @@ -165,33 +162,29 @@ }); let highestLoadedChunk = $derived.by(() => { - if (isLegacyMode || isSimpleMode) return 0; + if (isLegacyMode) return 0; const keys = Object.keys(loadedChunks).map(Number); if (keys.length === 0) return 0; return Math.max(...keys); }); let loadedRowsCount = $derived( - isLegacyMode || isSimpleMode - ? 0 - : Object.values(loadedChunks).reduce((sum, chunk) => sum + chunk.length, 0) + isLegacyMode ? 0 : Object.values(loadedChunks).reduce((sum, chunk) => sum + chunk.length, 0) ); let virtualTotalRows = $derived( - isLegacyMode || isSimpleMode + isLegacyMode ? 0 : Math.min(totalRows, Math.max(loadedRowsCount, (highestLoadedChunk + 1) * CHUNK_SIZE + 10)) ); - let startIndex = $derived( - isLegacyMode || isSimpleMode ? 0 : Math.max(0, Math.floor(scrollTop / ROW_HEIGHT) - 5) - ); + let startIndex = $derived(isLegacyMode ? 0 : Math.max(0, Math.floor(scrollTop / ROW_HEIGHT) - 5)); let endIndex = $derived( - isLegacyMode || isSimpleMode + isLegacyMode ? 0 : Math.min(virtualTotalRows, Math.floor((scrollTop + containerHeight) / ROW_HEIGHT) + 15) ); let visibleRows = $derived.by(() => { - if (isLegacyMode || isSimpleMode) return []; + if (isLegacyMode) return []; const rowsToRender: { index: number; data: unknown[] | null }[] = []; for (let i = startIndex; i < endIndex; i++) { const chunkIdx = Math.floor(i / CHUNK_SIZE); @@ -207,7 +200,7 @@ }); $effect(() => { - if (isLegacyMode || isSimpleMode || !fetchRows || hidden) return; + if (isLegacyMode || !fetchRows || hidden) return; const startChunk = Math.floor(startIndex / CHUNK_SIZE); const endChunk = Math.floor(endIndex / CHUNK_SIZE); @@ -236,17 +229,11 @@ }); }); - const isTruncated = $derived( - isLegacyMode - ? textTruncated - : isSimpleMode - ? totalRows > initialRows.length - : totalRows > loadedRowsCount - ); + const isTruncated = $derived(isLegacyMode ? textTruncated : totalRows > loadedRowsCount); const formattedLoadedRowsCount = $derived(loadedRowsCount.toLocaleString(getLocale())); $effect(() => { - if (!isLegacyMode && !isSimpleMode) { + if (!isLegacyMode) { showingRowsCount = loadedRowsCount; } }); @@ -289,45 +276,6 @@ {/if} {/if} -{:else if isSimpleMode} -
- {#if headers.length === 0} -

{m.storage_bucket_empty()}

- {:else} -
- - - - - {#each headers as header (header)} - - {/each} - - - - {#each initialRows as row, i (i)} - - - - {#each headers as _h, j (j)} - - {/each} - - {/each} - -
{header}
{(i + 1).toLocaleString(getLocale())}{row[j] !== null && row[j] !== undefined ? String(row[j]) : ''}
-
- {#if isTruncated} -

- {m.storage_preview_infinite_scroll_disabled()} -

- {/if} - {/if} -
{:else}
{#if headers.length === 0} diff --git a/src/lib/components/storage/modals/preview/ParquetPreview.svelte b/src/lib/components/storage/modals/preview/ParquetPreview.svelte index 68aee3b3..d0fc45e4 100644 --- a/src/lib/components/storage/modals/preview/ParquetPreview.svelte +++ b/src/lib/components/storage/modals/preview/ParquetPreview.svelte @@ -9,7 +9,7 @@ initialRows: unknown[][]; totalRows?: number; - fetchRows?: ( + fetchRows: ( offset: number, limit: number, onColumn?: (name: string, values: unknown[]) => void @@ -27,10 +27,6 @@ hidden = false }: Props = $props(); - const isSimpleMode = $derived(!fetchRows); - - // ── Virtual scroll mode: chunked data loading ── - const CHUNK_SIZE = PREVIEW_CHUNK_ROWS; const ROW_HEIGHT = 32; @@ -73,11 +69,10 @@ } onMount(() => { - if (!isSimpleMode) populateChunk0(); + populateChunk0(); }); $effect(() => { - if (isSimpleMode) return; if (initialRows && initialRows.length > 0 && !hidden) { untrack(() => { populateChunk0(); @@ -86,34 +81,26 @@ }); let highestLoadedChunk = $derived.by(() => { - if (isSimpleMode) return 0; const keys = Object.keys(loadedChunks).map(Number); if (keys.length === 0) return 0; return Math.max(...keys); }); let loadedRowsCount = $derived( - isSimpleMode ? 0 : Object.values(loadedChunks).reduce((sum, chunk) => sum + chunk.length, 0) + Object.values(loadedChunks).reduce((sum, chunk) => sum + chunk.length, 0) ); let virtualTotalRows = $derived( - isSimpleMode - ? 0 - : Math.min(totalRows, Math.max(loadedRowsCount, (highestLoadedChunk + 1) * CHUNK_SIZE + 10)) + Math.min(totalRows, Math.max(loadedRowsCount, (highestLoadedChunk + 1) * CHUNK_SIZE + 10)) ); - let startIndex = $derived(isSimpleMode ? 0 : Math.max(0, Math.floor(scrollTop / ROW_HEIGHT) - 5)); + let startIndex = $derived(Math.max(0, Math.floor(scrollTop / ROW_HEIGHT) - 5)); let endIndex = $derived( - isSimpleMode - ? 0 - : Math.min(virtualTotalRows, Math.floor((scrollTop + containerHeight) / ROW_HEIGHT) + 15) + Math.min(virtualTotalRows, Math.floor((scrollTop + containerHeight) / ROW_HEIGHT) + 15) ); - let paddingTop = $derived(isSimpleMode ? 0 : startIndex * ROW_HEIGHT); - let paddingBottom = $derived( - isSimpleMode ? 0 : Math.max(0, (virtualTotalRows - endIndex) * ROW_HEIGHT) - ); + let paddingTop = $derived(startIndex * ROW_HEIGHT); + let paddingBottom = $derived(Math.max(0, (virtualTotalRows - endIndex) * ROW_HEIGHT)); let visibleRows = $derived.by(() => { - if (isSimpleMode) return []; const rowsToRender: { index: number; data: unknown[] | null }[] = []; for (let i = startIndex; i < endIndex; i++) { const chunkIdx = Math.floor(i / CHUNK_SIZE); @@ -129,7 +116,7 @@ }); $effect(() => { - if (isSimpleMode || !fetchRows || hidden) return; + if (hidden) return; const currentStart = startIndex; const currentEnd = endIndex; @@ -179,45 +166,57 @@ }); }); - const isTruncated = $derived( - isSimpleMode ? totalRows > initialRows.length : totalRows > loadedRowsCount - ); + const isTruncated = $derived(totalRows > loadedRowsCount); const formattedLoadedRowsCount = $derived(loadedRowsCount.toLocaleString(getLocale())); const formattedTotalRows = $derived(totalRows.toLocaleString(getLocale())); $effect(() => { - if (!isSimpleMode) { - showingRowsCount = loadedRowsCount; - } + showingRowsCount = loadedRowsCount; }); -{#if isSimpleMode} -
- {#if headers.length === 0} -

{m.storage_bucket_empty()}

- {:else} -
- - - - - {#each headers as header (header)} - - {/each} +
+ {#if headers.length === 0} +

{m.storage_bucket_empty()}

+ {:else} + +
(scrollTop = e.currentTarget.scrollTop)} + > +
{header}
+ + + + {#each headers as header, j (header)} + + {/each} + + + + {#if paddingTop > 0} + + - - - {#each initialRows as row, i (i)} - - + {/if} + {#each visibleRows as row (row.index)} + + + {#if row.data !== null && row.data !== undefined} {#each headers as _h, j (j)} - {#if row[j] !== undefined} - {:else} {/if} {/each} - - {/each} - -
0 ? `width: ${columnWidths[j]}px` : ''}>{header}
{(i + 1).toLocaleString(getLocale())}
{(row.index + 1).toLocaleString(getLocale())} - {row[j] !== null ? String(row[j]) : ''} + {#if row.data[j] !== undefined} + + {row.data[j] !== null ? String(row.data[j]) : ''} @@ -225,96 +224,31 @@
-
- {#if isTruncated} -

- {m.storage_preview_infinite_scroll_disabled()} -

- {/if} - {/if} -
-{:else} -
- {#if headers.length === 0} -

{m.storage_bucket_empty()}

- {:else} - -
(scrollTop = e.currentTarget.scrollTop)} - > - - - - - {#each headers as header, j (header)} - - {/each} + {:else} + + {#each headers as _h, j (j)} + + {/each} + {/if} - - - {#if paddingTop > 0} - - - - {/if} - {#each visibleRows as row (row.index)} - - - {#if row.data !== null && row.data !== undefined} - - {#each headers as _h, j (j)} - {#if row.data[j] !== undefined} - - {:else} - - {/if} - {/each} - {:else} - - {#each headers as _h, j (j)} - - {/each} - {/if} - - {/each} - {#if paddingBottom > 0} - - - - {/if} - -
0 ? `width: ${columnWidths[j]}px` : ''}>{header} +
+
{(row.index + 1).toLocaleString(getLocale())} - {row.data[j] !== null ? String(row.data[j]) : ''} - -
- 		-
-
-
- {#if isTruncated} -

- {m.storage_preview_parquet_rows - ? m.storage_preview_parquet_rows({ - count: formattedLoadedRowsCount, - total: formattedTotalRows - }) - : `Showing ${formattedLoadedRowsCount} of ${formattedTotalRows} rows. Scroll to load more.`} -

- {/if} + {/each} + {#if paddingBottom > 0} + + + + {/if} + + +
+ {#if isTruncated} +

+ {m.storage_preview_parquet_rows({ + count: formattedLoadedRowsCount, + total: formattedTotalRows + })} +

{/if} -
-{/if} + {/if} + diff --git a/src/lib/components/storage/modals/preview/ParquetPreview.svelte.spec.ts b/src/lib/components/storage/modals/preview/ParquetPreview.svelte.spec.ts index 7f8622f2..c5387c67 100644 --- a/src/lib/components/storage/modals/preview/ParquetPreview.svelte.spec.ts +++ b/src/lib/components/storage/modals/preview/ParquetPreview.svelte.spec.ts @@ -10,9 +10,10 @@ describe('ParquetPreview', () => { ['2', 'Bob', 'bob@example.com'], ['3', 'Charlie', 'charlie@example.com'] ]; + const noopFetchRows = () => Promise.resolve([]); it('should show empty state when headers array is empty', async () => { - render(ParquetPreview, { headers: [], initialRows: [] }); + render(ParquetPreview, { headers: [], initialRows: [], fetchRows: noopFetchRows }); await expect.element(page.getByText('This bucket is empty')).toBeInTheDocument(); }); @@ -20,7 +21,8 @@ describe('ParquetPreview', () => { it('should render a table with headers', async () => { render(ParquetPreview, { headers: defaultHeaders, - initialRows: defaultRows + initialRows: defaultRows, + fetchRows: noopFetchRows }); const table = page.getByRole('table', { name: 'Parquet preview' }); @@ -35,7 +37,8 @@ describe('ParquetPreview', () => { render(ParquetPreview, { headers: defaultHeaders, initialRows: defaultRows, - totalRows: 3 + totalRows: 3, + fetchRows: noopFetchRows }); await expect @@ -53,7 +56,8 @@ describe('ParquetPreview', () => { render(ParquetPreview, { headers: defaultHeaders, initialRows: defaultRows, - totalRows: 1000 + totalRows: 1000, + fetchRows: noopFetchRows }); // With default CHUNK_SIZE=250, loadedRowsCount = 250 (one chunk) @@ -65,7 +69,8 @@ describe('ParquetPreview', () => { render(ParquetPreview, { headers: defaultHeaders, initialRows: defaultRows, - totalRows: 3 + totalRows: 3, + fetchRows: noopFetchRows }); // Only 3 rows loaded, totalRows = 3 — not truncated @@ -79,7 +84,8 @@ describe('ParquetPreview', () => { render(ParquetPreview, { headers: defaultHeaders, initialRows: defaultRows, - totalRows: 0 + totalRows: 0, + fetchRows: noopFetchRows }); // 0 total rows, no truncation message @@ -91,7 +97,8 @@ describe('ParquetPreview', () => { render(ParquetPreview, { headers: defaultHeaders, initialRows: rowsWithNull, - totalRows: 1 + totalRows: 1, + fetchRows: noopFetchRows }); // Alice should be visible @@ -156,7 +163,8 @@ describe('ParquetPreview', () => { render(ParquetPreview, { headers: defaultHeaders, initialRows: manyRows, - totalRows: 1000 + totalRows: 1000, + fetchRows: noopFetchRows }); await expect.element(page.getByText('User-250')).toBeInTheDocument(); @@ -166,7 +174,8 @@ describe('ParquetPreview', () => { const specialHeaders = ['first name', 'last.name', 'email-address']; render(ParquetPreview, { headers: specialHeaders, - initialRows: [['John', 'Doe', 'john@example.com']] + initialRows: [['John', 'Doe', 'john@example.com']], + fetchRows: noopFetchRows }); for (const header of specialHeaders) { @@ -178,7 +187,8 @@ describe('ParquetPreview', () => { render(ParquetPreview, { headers: defaultHeaders, initialRows: [], - totalRows: 0 + totalRows: 0, + fetchRows: noopFetchRows }); const table = page.getByRole('table', { name: 'Parquet preview' }); diff --git a/src/lib/components/storage/modals/upload/UploadModal.svelte b/src/lib/components/storage/modals/upload/UploadModal.svelte index 883d37f1..46e5c532 100644 --- a/src/lib/components/storage/modals/upload/UploadModal.svelte +++ b/src/lib/components/storage/modals/upload/UploadModal.svelte @@ -14,7 +14,7 @@ import { StorageError } from '$lib/storage/errors.js'; import { formatFileSize, keyToName } from '$lib/storage/utils.js'; import { connectionStore } from '$lib/storage/connection-store.svelte.js'; - import { uploadConcurrency } from '$lib/client/feature-flags.js'; + import { uploadConcurrency } from '$lib/client/config.js'; import UploadDropzone from './UploadDropzone.svelte'; import UploadEntryStatus from './UploadEntryStatus.svelte'; import ConflictEntry from '../shared/ConflictEntry.svelte'; diff --git a/src/lib/env-schema.test.ts b/src/lib/env-schema.test.ts new file mode 100644 index 00000000..df99abe0 --- /dev/null +++ b/src/lib/env-schema.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { envBoolean, envInteger, envList, envOptionalInteger, parseEnv } from './env-schema.js'; + +describe('envBoolean', () => { + const schema = envBoolean(false); + + it('parses true and false case-insensitively', () => { + expect(schema.parse('true')).toBe(true); + expect(schema.parse('FALSE')).toBe(false); + }); + + it('uses the default when unset or blank', () => { + expect(schema.parse(undefined)).toBe(false); + expect(schema.parse(' ')).toBe(false); + }); + + it('rejects other values', () => { + expect(schema.safeParse('yes').success).toBe(false); + expect(schema.safeParse('1').success).toBe(false); + }); +}); + +describe('envInteger', () => { + const schema = envInteger(8, { max: 10 }); + + it('parses a trimmed integer and uses the default when blank', () => { + expect(schema.parse(' 4 ')).toBe(4); + expect(schema.parse('')).toBe(8); + }); + + it('rejects non-integers and values out of range', () => { + for (const value of ['1.5', '8x', '0', '11']) { + expect(schema.safeParse(value).success, value).toBe(false); + } + }); +}); + +describe('envOptionalInteger', () => { + it('is undefined when unset', () => { + expect(envOptionalInteger.parse(undefined)).toBeUndefined(); + }); +}); + +describe('envList', () => { + it('splits, trims and drops empty entries', () => { + expect(envList(['a']).parse(' x , ,y ')).toEqual(['x', 'y']); + }); + + it('uses the default when unset', () => { + expect(envList(['a']).parse(undefined)).toEqual(['a']); + }); +}); + +describe('parseEnv', () => { + const schema = z.object({ A: envInteger(1), B: envBoolean(true) }); + + it('returns the parsed values', () => { + expect(parseEnv(schema, { A: '2' })).toEqual({ A: 2, B: true }); + }); + + it('names every invalid variable in one error', () => { + expect(() => parseEnv(schema, { A: 'x', B: 'y' })).toThrow(/A: .*\n- B: /s); + }); +}); diff --git a/src/lib/env-schema.ts b/src/lib/env-schema.ts new file mode 100644 index 00000000..4a40f54d --- /dev/null +++ b/src/lib/env-schema.ts @@ -0,0 +1,72 @@ +// zod building blocks for configuration read from environment variables. +// Kept free of SvelteKit and server imports: both client and server config use it. + +import { z } from 'zod'; + +type Env = Record; + +/** Treats an empty or whitespace-only value like an unset variable. */ +function blankAsUnset(value: unknown): unknown { + if (typeof value !== 'string') return value; + const trimmed = value.trim(); + return trimmed === '' ? undefined : trimmed; +} + +/** `true` or `false` (case-insensitive). */ +export function envBoolean(defaultValue: boolean) { + return z.preprocess( + blankAsUnset, + z.stringbool({ truthy: ['true'], falsy: ['false'] }).default(defaultValue) + ); +} + +/** An integer within `min` and `max`. */ +export function envInteger( + defaultValue: number, + { min = 1, max = Number.MAX_SAFE_INTEGER }: { min?: number; max?: number } = {} +) { + return z.preprocess( + blankAsUnset, + z.coerce.number().int().min(min).max(max).default(defaultValue) + ); +} + +/** A positive integer, or `undefined` when unset. */ +export const envOptionalInteger = z.preprocess( + blankAsUnset, + z.coerce.number().int().positive().optional() +); + +/** A trimmed string, or `undefined` when unset. */ +export const envString = z.preprocess(blankAsUnset, z.string().optional()); + +/** A list of non-empty entries separated by `separator`. */ +export function envList(defaultValue: string[], separator: RegExp | string = ',') { + return z.preprocess( + blankAsUnset, + z + .string() + .transform((value) => + value + .split(separator) + .map((entry) => entry.trim()) + .filter(Boolean) + ) + .default(defaultValue) + ); +} + +/** + * Parses `env` with `schema`. Throws one error that names every invalid + * variable, so a misconfigured deployment fails at startup with all problems + * listed instead of falling back to defaults. + */ +export function parseEnv(schema: T, env: Env): z.output { + const result = schema.safeParse(env); + if (result.success) return result.data; + const problems = result.error.issues.map((issue) => { + const name = issue.path.join('.'); + return name ? `${name}: ${issue.message}` : issue.message; + }); + throw new Error(`Invalid configuration:\n- ${problems.join('\n- ')}`); +} diff --git a/src/lib/server/feature-flags.test.ts b/src/lib/server/feature-flags.test.ts deleted file mode 100644 index 78223427..00000000 --- a/src/lib/server/feature-flags.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; - -vi.mock('$lib/server/logging', () => import('$lib/test-utils/mock-logger.js')); - -import { parseParquetDisallowed, resolveArchiveCacheMaxBytes } from './feature-flags.js'; - -const MB = 1024 * 1024; - -describe('resolveArchiveCacheMaxBytes', () => { - it('keeps a limit of at least twice the preview limit', () => { - expect(resolveArchiveCacheMaxBytes(512 * MB, 100 * MB)).toBe(512 * MB); - expect(resolveArchiveCacheMaxBytes(200 * MB, 100 * MB)).toBe(200 * MB); - }); - - it('raises a lower limit to twice the preview limit', () => { - expect(resolveArchiveCacheMaxBytes(150 * MB, 100 * MB)).toBe(200 * MB); - }); -}); - -describe('parseParquetDisallowed', () => { - it('defaults to GZIP with requireOffsetIndex when value is undefined', () => { - expect(parseParquetDisallowed(undefined)).toEqual([ - { codec: 'GZIP', requireOffsetIndex: true } - ]); - }); - - it('defaults to GZIP with requireOffsetIndex when value is empty string', () => { - expect(parseParquetDisallowed('')).toEqual([{ codec: 'GZIP', requireOffsetIndex: true }]); - }); - - it('parses a single codec without suffix', () => { - expect(parseParquetDisallowed('zstd')).toEqual([{ codec: 'ZSTD', requireOffsetIndex: false }]); - }); - - it('parses a codec with -no_offset suffix', () => { - expect(parseParquetDisallowed('gzip-no_offset')).toEqual([ - { codec: 'GZIP', requireOffsetIndex: true } - ]); - }); - - it('parses comma-separated list with mixed suffixes', () => { - expect(parseParquetDisallowed('gzip-no_offset,zstd,snappy-no_offset')).toEqual([ - { codec: 'GZIP', requireOffsetIndex: true }, - { codec: 'ZSTD', requireOffsetIndex: false }, - { codec: 'SNAPPY', requireOffsetIndex: true } - ]); - }); - - it('converts codec names to uppercase', () => { - expect(parseParquetDisallowed('Gzip,Zstd')).toEqual([ - { codec: 'GZIP', requireOffsetIndex: false }, - { codec: 'ZSTD', requireOffsetIndex: false } - ]); - }); - - it('filters out empty entries', () => { - expect(parseParquetDisallowed('gzip,,zstd')).toEqual([ - { codec: 'GZIP', requireOffsetIndex: false }, - { codec: 'ZSTD', requireOffsetIndex: false } - ]); - }); - - it('trims whitespace', () => { - expect(parseParquetDisallowed(' gzip , zstd ')).toEqual([ - { codec: 'GZIP', requireOffsetIndex: false }, - { codec: 'ZSTD', requireOffsetIndex: false } - ]); - }); -}); diff --git a/src/lib/server/feature-flags.ts b/src/lib/server/feature-flags.ts index c819a959..cc5d911c 100644 --- a/src/lib/server/feature-flags.ts +++ b/src/lib/server/feature-flags.ts @@ -1,167 +1,32 @@ // Server-side read of cross-cutting feature flags. Read once at module init // from the live process env so consumers don't repeat the env-name + parsing. +// Limits and other settings live in the config modules, not here. +import { z } from 'zod'; import { env } from '$env/dynamic/private'; -import { logger } from '$lib/server/logging'; - -const log = logger.child({ module: 'feature-flags' }); +import { envBoolean, parseEnv } from '$lib/env-schema.js'; + +const flags = parseEnv( + z.object({ + STACKABLE_COCKPIT_COMPLETION_ENABLED: envBoolean(true), + STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED: envBoolean(false) + }), + env +); /** When `STACKABLE_COCKPIT_COMPLETION_ENABLED=false`, the SQL editor's * code-completion provider is not registered and the metadata endpoint * refuses requests. Useful to fall back to plain syntax highlighting if * completion misbehaves or generates undesirable load on Trino. */ -export const completionEnabled = env.STACKABLE_COCKPIT_COMPLETION_ENABLED !== 'false'; +export const completionEnabled = flags.STACKABLE_COCKPIT_COMPLETION_ENABLED; /** When `STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED=true`, the S3/HDFS file * browser is shown in the sidebar and routes under `/storage` become * active. Disabled by default — opt in explicitly to expose storage * credentials and the file-browser UI. */ -export const storageBrowserEnabled = env.STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED === 'true'; - -// ── Storage preview limits ─────────────────────────────────────────────────── - -/** Maximum bytes fetched when streaming a text, CSV, or JSON file preview. - * Controlled by `STACKABLE_COCKPIT_TEXT_PREVIEW_BYTES`. Default: 262144 (256 KiB). - * Larger values allow more of a file to be visible in-browser but increase - * memory pressure and response latency. Only applies to files larger than - * `maxEditableFileSize`; editable files are always fetched completely. */ -export const textPreviewBytes = - parseInt(env.STACKABLE_COCKPIT_TEXT_PREVIEW_BYTES ?? '', 10) || 256 * 1024; - -/** Maximum bytes fetched when streaming an image file preview. - * Controlled by `STACKABLE_COCKPIT_IMAGE_PREVIEW_BYTES`. Default: 5242880 (5 MiB). - * Raise for large high-resolution images; lower to reduce bandwidth on - * deployments where storage egress is expensive. */ -export const imagePreviewBytes = - parseInt(env.STACKABLE_COCKPIT_IMAGE_PREVIEW_BYTES ?? '', 10) || 5 * 1024 * 1024; - -/** Maximum bytes fetched when streaming a PDF file preview. - * Controlled by `STACKABLE_COCKPIT_PDF_PREVIEW_BYTES`. Default: 26214400 (25 MiB). - * PDFs typically need several megabytes for the initial pages to render - * correctly in the browser PDF viewer. */ -export const pdfPreviewBytes = - parseInt(env.STACKABLE_COCKPIT_PDF_PREVIEW_BYTES ?? '', 10) || 25 * 1024 * 1024; - -/** Maximum number of rows included in a file preview that is rendered as a - * table (currently: Parquet files converted to CSV). - * Controlled by `STACKABLE_COCKPIT_FILE_PREVIEW_ROWS`. Default: 250. - * Higher values give more data context but increase server-side S3 reads - * and the size of the payload sent to the browser. */ -export const filePreviewRows = parseInt(env.STACKABLE_COCKPIT_FILE_PREVIEW_ROWS ?? '', 10) || 250; - -/** Maximum number of columns included in a file preview that is rendered as a - * table (currently: used by CSV). - * Controlled by `STACKABLE_COCKPIT_FILE_PREVIEW_COLUMNS`. Default: 50. - * Values only affect client side rendering and not payload size. */ -export const filePreviewColumns = - parseInt(env.STACKABLE_COCKPIT_FILE_PREVIEW_COLUMNS ?? '', 10) || 50; - -/** Maximum compressed size of an archive that will be opened for in-browser - * preview. Archives larger than this threshold will not be downloaded at all - * and a "too large" fallback is shown instead. During listing the total - * decompressed entry size is also checked against this limit. - * Controlled by `STACKABLE_COCKPIT_ARCHIVE_PREVIEW_MAX_MB`. Default: 100 MB. */ -export const archivePreviewMaxBytes = - parseInt(env.STACKABLE_COCKPIT_ARCHIVE_PREVIEW_MAX_MB ?? '', 10) * 1024 * 1024 || - 100 * 1024 * 1024; - -/** Maximum total size of the archive files cached on local disk (`/tmp`) for - * archive previews, nested archives included. When the cache is full, the - * least recently used idle archive is evicted; if every cached archive is in - * use, the request is rejected. Never lower than twice `archivePreviewMaxBytes`: - * a nested archive preview keeps the outer archive cached while the nested one - * is written, so it needs room for both. - * Controlled by `STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB`. Default: 512 MB. */ -export const archiveCacheMaxBytes = resolveArchiveCacheMaxBytes( - parseInt(env.STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB ?? '', 10) * 1024 * 1024 || 512 * 1024 * 1024, - archivePreviewMaxBytes -); - -export function resolveArchiveCacheMaxBytes(configured: number, previewMaxBytes: number): number { - const minimum = 2 * previewMaxBytes; - if (configured >= minimum) return configured; - log.warn( - { configured_bytes: configured, minimum_bytes: minimum }, - 'archive cache limit is below twice the archive preview limit, raising it' - ); - return minimum; -} - -// ── Parquet preview restrictions ─────────────────────────────────────────── - -export interface ParquetDisallowedCompression { - /** Upper-case compression codec name (e.g. `GZIP`, `ZSTD`, `SNAPPY`). */ - codec: string; - /** When true, only block files using this codec if they lack an offset index. */ - requireOffsetIndex: boolean; -} - -export function parseParquetDisallowed(value: string | undefined): ParquetDisallowedCompression[] { - if (!value) return [{ codec: 'GZIP', requireOffsetIndex: true }]; - return value - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - .map((entry) => { - if (entry.endsWith('-no_offset')) { - return { - codec: entry.slice(0, -'-no_offset'.length).toUpperCase(), - requireOffsetIndex: true - }; - } - return { codec: entry.toUpperCase(), requireOffsetIndex: false }; - }); -} - -/** Comma-separated list of compression types to disallow from parquet - * data preview. Each entry is either a compression codec name (e.g. `gzip`, - * `zstd`, `snappy`) or a codec suffixed with `-no_offset` (e.g. - * `gzip-no_offset`) to only block when the file lacks an offset index. - * Controlled by `STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES`. - * Default: `gzip-no_offset`. */ -export const parquetDisallowedCompression = parseParquetDisallowed( - env.STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES ?? 'gzip-no_offset' -); - -// ── Flags shared with the client ───────────────────────────────────────────── -// -// These are `PUBLIC_` settings that the server enforces too. They are defined -// once in `$lib/client/feature-flags` so both sides parse them identically. -// -// `maxEditableFileSize`: save requests above it are rejected with HTTP 413. -// -// Context actions: server-side enforcement (cost control, not authorisation) on -// `POST /api/storage/transfer` and its `/plan`, by the request's `op`: -// - `copy` requires paste. -// - `move` requires move or paste, because drag-and-drop and cut-and-paste -// both move. -// - `rename` requires rename. -// Cut/copy only fill the client clipboard and have no endpoint of their own. -export { - infiniteScrollEnabled, - maxEditableFileSize, - storagePasteEnabled, - storageRenameEnabled, - storageMoveEnabled -} from '$lib/client/feature-flags.js'; - -/** Objects copied at once within one transfer request (copy, move, rename). - * Controlled by `STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY`. Default: 8; - * a missing or invalid value falls back to the default. */ -export const storageTransferConcurrency: number = (() => { - // Number() rather than parseInt, so "8x" or "1.5" fall back instead of being truncated. - const parsed = Number(env.STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY?.trim() || NaN); - return Number.isInteger(parsed) && parsed > 0 ? parsed : 8; -})(); - -// ── Storage browser: downloads ───────────────────────────────────────────── +export const storageBrowserEnabled = flags.STACKABLE_COCKPIT_STORAGE_BROWSER_ENABLED; -/** Days that a user's download history remains available. Controlled by - * `STACKABLE_COCKPIT_DOWNLOAD_HISTORY_RETENTION_DAYS`. */ -const retentionDays = Number(env.STACKABLE_COCKPIT_DOWNLOAD_HISTORY_RETENTION_DAYS ?? 30); -if (!Number.isInteger(retentionDays) || retentionDays < 1 || retentionDays > 3650) { - throw new Error( - 'STACKABLE_COCKPIT_DOWNLOAD_HISTORY_RETENTION_DAYS must be an integer between 1 and 3650' - ); -} -export const downloadHistoryRetentionMs = retentionDays * 24 * 60 * 60 * 1000; +// Shared with the client and enforced by `POST /api/storage/transfer` and its +// `/plan`. Cut and copy only fill the client clipboard and have no endpoint of +// their own. +export { storageTransferEnabled } from '$lib/client/feature-flags.js'; diff --git a/src/lib/server/storage/archive.test.ts b/src/lib/server/storage/archive.test.ts index dd366a5a..8920d246 100644 --- a/src/lib/server/storage/archive.test.ts +++ b/src/lib/server/storage/archive.test.ts @@ -13,7 +13,7 @@ vi.mock('$lib/server/logging', () => import('$lib/test-utils/mock-logger.js')); const DEFAULT_CACHE_MAX_BYTES = 512 * 1024 * 1024; const flags = vi.hoisted(() => ({ archiveCacheMaxBytes: 512 * 1024 * 1024 })); -vi.mock('$lib/server/feature-flags.js', () => ({ +vi.mock('$lib/server/storage/config.js', () => ({ get archiveCacheMaxBytes() { return flags.archiveCacheMaxBytes; } diff --git a/src/lib/server/storage/archive.ts b/src/lib/server/storage/archive.ts index 1b19b9a8..dd6cef11 100644 --- a/src/lib/server/storage/archive.ts +++ b/src/lib/server/storage/archive.ts @@ -19,7 +19,7 @@ import { createGunzip } from 'node:zlib'; import AdmZip from 'adm-zip'; import * as tar from 'tar-stream'; import { logger } from '$lib/server/logging'; -import { archiveCacheMaxBytes } from '$lib/server/feature-flags.js'; +import { archiveCacheMaxBytes } from '$lib/server/storage/config.js'; import { storageArchiveCacheBytes, storageArchiveCacheEvictionsTotal diff --git a/src/lib/server/storage/config.test.ts b/src/lib/server/storage/config.test.ts new file mode 100644 index 00000000..b38c0c2d --- /dev/null +++ b/src/lib/server/storage/config.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('$lib/server/logging', () => import('$lib/test-utils/mock-logger.js')); + +import { parseStorageConfig } from './config.js'; + +const MB = 1024 * 1024; + +describe('parseStorageConfig', () => { + it('uses the defaults when nothing is set', () => { + const config = parseStorageConfig({}); + expect(config.archivePreviewMaxBytes).toBe(100 * MB); + expect(config.archiveCacheMaxBytes).toBe(512 * MB); + expect(config.downloadHistoryRetentionMs).toBe(30 * 24 * 60 * 60 * 1000); + expect(config.parquetDisallowedCompression).toEqual([ + { codec: 'GZIP', requireOffsetIndex: true } + ]); + expect(config.tlsCaCertPath).toBeUndefined(); + }); + + it('parses Parquet codecs with and without the -no_offset suffix', () => { + const config = parseStorageConfig({ + STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES: 'zstd, Snappy-no_offset' + }); + expect(config.parquetDisallowedCompression).toEqual([ + { codec: 'ZSTD', requireOffsetIndex: false }, + { codec: 'SNAPPY', requireOffsetIndex: true } + ]); + }); + + it('limits the download history retention to 3650 days', () => { + expect(() => + parseStorageConfig({ STACKABLE_COCKPIT_DOWNLOAD_HISTORY_RETENTION_DAYS: '3651' }) + ).toThrow(/STACKABLE_COCKPIT_DOWNLOAD_HISTORY_RETENTION_DAYS/); + }); + + it('raises the archive cache to twice the archive preview limit', () => { + const config = parseStorageConfig({ + STACKABLE_COCKPIT_ARCHIVE_PREVIEW_MAX_MB: '300', + STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB: '512' + }); + expect(config.archiveCacheMaxBytes).toBe(600 * MB); + }); +}); diff --git a/src/lib/server/storage/config.ts b/src/lib/server/storage/config.ts new file mode 100644 index 00000000..1f5d19cd --- /dev/null +++ b/src/lib/server/storage/config.ts @@ -0,0 +1,149 @@ +// Storage browser settings: limits and tuning, not feature toggles (those are +// in `$lib/server/feature-flags`). Parsed once at module init. + +import { z } from 'zod'; +import { env } from '$env/dynamic/private'; +import { envInteger, envList, envString, parseEnv } from '$lib/env-schema.js'; +import { logger } from '$lib/server/logging'; + +const log = logger.child({ module: 'storage-config' }); + +const MB = 1024 * 1024; + +export interface ParquetDisallowedCompression { + /** Upper-case compression codec name (e.g. `GZIP`, `ZSTD`, `SNAPPY`). */ + codec: string; + /** When true, only block files using this codec if they lack an offset index. */ + requireOffsetIndex: boolean; +} + +function toParquetDisallowed(entry: string): ParquetDisallowedCompression { + if (entry.endsWith('-no_offset')) { + return { codec: entry.slice(0, -'-no_offset'.length).toUpperCase(), requireOffsetIndex: true }; + } + return { codec: entry.toUpperCase(), requireOffsetIndex: false }; +} + +const schema = z.object({ + STACKABLE_COCKPIT_TEXT_PREVIEW_BYTES: envInteger(256 * 1024), + STACKABLE_COCKPIT_IMAGE_PREVIEW_BYTES: envInteger(5 * MB), + STACKABLE_COCKPIT_PDF_PREVIEW_BYTES: envInteger(25 * MB), + STACKABLE_COCKPIT_FILE_PREVIEW_ROWS: envInteger(250), + STACKABLE_COCKPIT_FILE_PREVIEW_COLUMNS: envInteger(50), + STACKABLE_COCKPIT_ARCHIVE_PREVIEW_MAX_MB: envInteger(100), + STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB: envInteger(512), + STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES: envList(['gzip-no_offset']), + STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY: envInteger(8), + STACKABLE_COCKPIT_DOWNLOAD_HISTORY_RETENTION_DAYS: envInteger(30, { max: 3650 }), + STACKABLE_COCKPIT_STORAGE_TLS_CA_CERT: envString +}); + +function resolveArchiveCacheMaxBytes(configured: number, previewMaxBytes: number): number { + const minimum = 2 * previewMaxBytes; + if (configured >= minimum) return configured; + log.warn( + { configured_bytes: configured, minimum_bytes: minimum }, + 'archive cache limit is below twice the archive preview limit, raising it' + ); + return minimum; +} + +export function parseStorageConfig(env: Record) { + const config = parseEnv(schema, env); + const archivePreviewMaxBytes = config.STACKABLE_COCKPIT_ARCHIVE_PREVIEW_MAX_MB * MB; + return { + textPreviewBytes: config.STACKABLE_COCKPIT_TEXT_PREVIEW_BYTES, + imagePreviewBytes: config.STACKABLE_COCKPIT_IMAGE_PREVIEW_BYTES, + pdfPreviewBytes: config.STACKABLE_COCKPIT_PDF_PREVIEW_BYTES, + filePreviewRows: config.STACKABLE_COCKPIT_FILE_PREVIEW_ROWS, + filePreviewColumns: config.STACKABLE_COCKPIT_FILE_PREVIEW_COLUMNS, + archivePreviewMaxBytes, + archiveCacheMaxBytes: resolveArchiveCacheMaxBytes( + config.STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB * MB, + archivePreviewMaxBytes + ), + parquetDisallowedCompression: + config.STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES.map( + toParquetDisallowed + ), + storageTransferConcurrency: config.STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY, + downloadHistoryRetentionMs: + config.STACKABLE_COCKPIT_DOWNLOAD_HISTORY_RETENTION_DAYS * 24 * 60 * 60 * 1000, + tlsCaCertPath: config.STACKABLE_COCKPIT_STORAGE_TLS_CA_CERT + }; +} + +const config = parseStorageConfig(env); + +// ── Preview limits ─────────────────────────────────────────────────────────── + +/** Maximum bytes fetched when streaming a text, CSV, or JSON file preview. + * Only applies to files larger than `maxEditableFileSize`; editable files are + * always fetched completely. + * Controlled by `STACKABLE_COCKPIT_TEXT_PREVIEW_BYTES`. Default: 262144 (256 KiB). */ +export const textPreviewBytes = config.textPreviewBytes; + +/** Maximum bytes fetched when streaming an image file preview. + * Controlled by `STACKABLE_COCKPIT_IMAGE_PREVIEW_BYTES`. Default: 5242880 (5 MiB). */ +export const imagePreviewBytes = config.imagePreviewBytes; + +/** Maximum bytes fetched when streaming a PDF file preview. PDFs typically need + * several megabytes for the initial pages to render in the browser PDF viewer. + * Controlled by `STACKABLE_COCKPIT_PDF_PREVIEW_BYTES`. Default: 26214400 (25 MiB). */ +export const pdfPreviewBytes = config.pdfPreviewBytes; + +/** Maximum number of rows included in a file preview that is rendered as a + * table (currently: Parquet files converted to CSV). + * Controlled by `STACKABLE_COCKPIT_FILE_PREVIEW_ROWS`. Default: 250. */ +export const filePreviewRows = config.filePreviewRows; + +/** Maximum number of columns included in a file preview that is rendered as a + * table (currently: used by CSV). Only affects client-side rendering, not the + * payload size. + * Controlled by `STACKABLE_COCKPIT_FILE_PREVIEW_COLUMNS`. Default: 50. */ +export const filePreviewColumns = config.filePreviewColumns; + +/** Maximum compressed size of an archive that will be opened for in-browser + * preview. Larger archives are not downloaded at all and a "too large" + * fallback is shown instead. During listing the total decompressed entry size + * is also checked against this limit. + * Controlled by `STACKABLE_COCKPIT_ARCHIVE_PREVIEW_MAX_MB`. Default: 100 MB. */ +export const archivePreviewMaxBytes = config.archivePreviewMaxBytes; + +/** Maximum total size of the archive files cached on local disk (`/tmp`) for + * archive previews, nested archives included. When the cache is full, the + * least recently used idle archive is evicted; if every cached archive is in + * use, the request is rejected. Never lower than twice `archivePreviewMaxBytes`: + * a nested archive preview keeps the outer archive cached while the nested one + * is written, so it needs room for both. + * Controlled by `STACKABLE_COCKPIT_ARCHIVE_CACHE_MAX_MB`. Default: 512 MB. */ +export const archiveCacheMaxBytes = config.archiveCacheMaxBytes; + +/** Compression types excluded from Parquet data preview. Each comma-separated + * entry is a codec name (e.g. `gzip`, `zstd`, `snappy`), or a codec suffixed + * with `-no_offset` (e.g. `gzip-no_offset`) to only block files that lack an + * offset index. + * Controlled by `STACKABLE_COCKPIT_PARQUET_PREVIEW_DISALLOWED_COMPRESSION_TYPES`. + * Default: `gzip-no_offset`. */ +export const parquetDisallowedCompression = config.parquetDisallowedCompression; + +// ── Transfers and downloads ────────────────────────────────────────────────── + +/** Objects copied at once within one transfer request (copy, move, rename). + * Controlled by `STACKABLE_COCKPIT_STORAGE_TRANSFER_CONCURRENCY`. Default: 8. */ +export const storageTransferConcurrency = config.storageTransferConcurrency; + +/** How long a user's download history remains available. Controlled by + * `STACKABLE_COCKPIT_DOWNLOAD_HISTORY_RETENTION_DAYS` (1 to 3650). Default: 30 days. */ +export const downloadHistoryRetentionMs = config.downloadHistoryRetentionMs; + +// ── Connections ────────────────────────────────────────────────────────────── + +/** Path to a CA certificate for S3 connections with TLS verification. It + * replaces the CAs bundled with Node.js. + * Controlled by `STACKABLE_COCKPIT_STORAGE_TLS_CA_CERT`. */ +export const storageTlsCaCertPath = config.tlsCaCertPath; + +// Shared with the client, which hides the save button above it; the save +// endpoint rejects larger saves with HTTP 413. +export { maxEditableFileSize } from '$lib/client/config.js'; diff --git a/src/lib/server/storage/download-manifests.ts b/src/lib/server/storage/download-manifests.ts index 4b543341..7dbdf2df 100644 --- a/src/lib/server/storage/download-manifests.ts +++ b/src/lib/server/storage/download-manifests.ts @@ -2,7 +2,7 @@ import { and, desc, eq, gt, lte, inArray } from 'drizzle-orm'; import { error, isHttpError } from '@sveltejs/kit'; import { z } from 'zod'; import { db } from '$lib/server/db.js'; -import { downloadHistoryRetentionMs } from '$lib/server/feature-flags.js'; +import { downloadHistoryRetentionMs } from '$lib/server/storage/config.js'; import { logger } from '$lib/server/logging'; import { storageDownloadManifests } from '$lib/server/schema.js'; import { getConnectionForUser } from './connections-db.js'; diff --git a/src/lib/server/storage/preview/csv.test.ts b/src/lib/server/storage/preview/csv.test.ts index 7a18f034..8bd955ca 100644 --- a/src/lib/server/storage/preview/csv.test.ts +++ b/src/lib/server/storage/preview/csv.test.ts @@ -5,9 +5,8 @@ import { createMockLogger } from '$lib/test-utils/mock-logger.js'; vi.mock('$lib/server/logging', () => import('$lib/test-utils/mock-logger.js')); // Use a small textPreviewBytes to force predictable chunk boundaries -vi.mock('$lib/server/feature-flags', () => ({ - textPreviewBytes: 11, - infiniteScrollEnabled: true +vi.mock('$lib/server/storage/config.js', () => ({ + textPreviewBytes: 11 })); import { getCsvPreview } from './csv.js'; diff --git a/src/lib/server/storage/preview/csv.ts b/src/lib/server/storage/preview/csv.ts index 1b969063..730f9225 100644 --- a/src/lib/server/storage/preview/csv.ts +++ b/src/lib/server/storage/preview/csv.ts @@ -1,6 +1,6 @@ import type pino from 'pino'; import type { StorageProvider } from '$lib/server/storage/provider.js'; -import { textPreviewBytes, infiniteScrollEnabled } from '$lib/server/feature-flags.js'; +import { textPreviewBytes } from '$lib/server/storage/config.js'; import { logger } from '$lib/server/logging'; import { ExpiringCache, objectVersion, previewCacheKey } from './cache.js'; @@ -188,7 +188,6 @@ export async function getCsvPreview( 'Content-Type': 'application/x-ndjson', 'X-Preview-Format': 'csv', 'X-Preview-Renderable': 'true', - 'X-Preview-Infinite-Scroll': String(infiniteScrollEnabled), 'X-Preview-Total-Size': '0', 'X-Preview-Total-Rows': '0', 'X-Preview-Offset': String(offset), @@ -241,7 +240,6 @@ export async function getCsvPreview( 'Content-Type': 'application/x-ndjson', 'X-Preview-Format': 'csv', 'X-Preview-Renderable': 'true', - 'X-Preview-Infinite-Scroll': String(infiniteScrollEnabled), 'X-Preview-Total-Size': String(totalSize), 'X-Preview-Total-Rows': String(entry.lineOffsets.length), 'X-Preview-Offset': String(offset), @@ -296,7 +294,6 @@ export async function getCsvPreview( 'Content-Type': 'application/x-ndjson', 'X-Preview-Format': 'csv', 'X-Preview-Renderable': 'true', - 'X-Preview-Infinite-Scroll': String(infiniteScrollEnabled), 'X-Preview-Truncated': String(truncated), 'X-Preview-Total-Size': String(totalSize), 'X-Preview-Total-Rows': String(entry.lineOffsets.length), diff --git a/src/lib/server/storage/preview/parquet.test.ts b/src/lib/server/storage/preview/parquet.test.ts index 3720144c..6355e0dc 100644 --- a/src/lib/server/storage/preview/parquet.test.ts +++ b/src/lib/server/storage/preview/parquet.test.ts @@ -18,15 +18,8 @@ vi.mock('hyparquet-compressors', () => ({ vi.mock('$lib/server/logging', () => import('$lib/test-utils/mock-logger.js')); -vi.mock('$lib/server/feature-flags', () => ({ - parquetDisallowedCompression: [{ codec: 'GZIP', requireOffsetIndex: true }], - storageBrowserEnabled: true, - completionEnabled: true, - filePreviewRows: 250, - textPreviewBytes: 256 * 1024, - imagePreviewBytes: 5 * 1024 * 1024, - pdfPreviewBytes: 25 * 1024 * 1024, - infiniteScrollEnabled: true +vi.mock('$lib/server/storage/config.js', () => ({ + parquetDisallowedCompression: [{ codec: 'GZIP', requireOffsetIndex: true }] })); import { getParquetPreview } from './parquet.js'; diff --git a/src/lib/server/storage/preview/parquet.ts b/src/lib/server/storage/preview/parquet.ts index b36bc57a..7a37b25a 100644 --- a/src/lib/server/storage/preview/parquet.ts +++ b/src/lib/server/storage/preview/parquet.ts @@ -13,9 +13,8 @@ import { logger } from '$lib/server/logging'; import type { StorageProvider } from '$lib/server/storage/provider.js'; import { parquetDisallowedCompression, - infiniteScrollEnabled, type ParquetDisallowedCompression -} from '$lib/server/feature-flags'; +} from '$lib/server/storage/config.js'; import { ExpiringCache, objectVersion, previewCacheKey, type ObjectVersionInfo } from './cache.js'; const fallbackLog = logger.child({ module: 'parquet-preview' }); @@ -448,7 +447,6 @@ export async function getParquetPreview( 'Content-Type': 'application/x-ndjson', 'X-Preview-Format': 'parquet', 'X-Preview-Renderable': 'true', - 'X-Preview-Infinite-Scroll': String(infiniteScrollEnabled), 'X-Preview-Data-Blocked': 'false', 'X-Preview-Total-Size': '0', 'X-Preview-Total-Rows': '0', @@ -525,7 +523,6 @@ export async function getParquetPreview( 'Content-Type': 'application/x-ndjson', 'X-Preview-Format': 'parquet', 'X-Preview-Renderable': 'true', - 'X-Preview-Infinite-Scroll': String(infiniteScrollEnabled), ...extraHeaders, 'X-Preview-Total-Size': String(byteLength), 'X-Preview-Total-Rows': String(totalRows), @@ -546,7 +543,6 @@ export async function getParquetPreview( 'Content-Type': 'application/x-ndjson', 'X-Preview-Format': 'parquet', 'X-Preview-Renderable': 'true', - 'X-Preview-Infinite-Scroll': String(infiniteScrollEnabled), 'X-Preview-Truncated': 'false', 'X-Preview-Total-Size': String(byteLength), 'X-Preview-Total-Rows': String(totalRows), @@ -715,7 +711,6 @@ export async function getParquetPreview( 'Content-Type': 'application/x-ndjson', 'X-Preview-Format': 'parquet', 'X-Preview-Renderable': 'true', - 'X-Preview-Infinite-Scroll': String(infiniteScrollEnabled), 'X-Preview-Truncated': String(truncated), 'X-Preview-Total-Size': String(byteLength), 'X-Preview-Total-Rows': String(totalRows), diff --git a/src/lib/server/storage/preview/stream.ts b/src/lib/server/storage/preview/stream.ts index c00dd511..a2d7ef05 100644 --- a/src/lib/server/storage/preview/stream.ts +++ b/src/lib/server/storage/preview/stream.ts @@ -7,7 +7,7 @@ import { filePreviewRows, filePreviewColumns, maxEditableFileSize -} from '$lib/server/feature-flags.js'; +} from '$lib/server/storage/config.js'; import { isEditableTextObject } from '$lib/storage/text-content-type.js'; /** diff --git a/src/lib/server/storage/s3-client.ts b/src/lib/server/storage/s3-client.ts index 5aba9972..0a00ada6 100644 --- a/src/lib/server/storage/s3-client.ts +++ b/src/lib/server/storage/s3-client.ts @@ -1,10 +1,10 @@ -import { env } from '$env/dynamic/private'; import { readFileSync } from 'node:fs'; import { S3Client } from '@aws-sdk/client-s3'; import { NodeHttpHandler } from '@smithy/node-http-handler'; import { Agent as HttpAgent } from 'http'; import { Agent as HttpsAgent } from 'https'; import { logger } from '$lib/server/logging'; +import { storageTlsCaCertPath } from './config.js'; import type { S3ConnectionConfig } from './types.js'; const log = logger.child({ module: 's3-client' }); @@ -14,7 +14,7 @@ const log = logger.child({ module: 's3-client' }); * bundled with Node.js, like the Trino and database CA settings. */ function loadCaCert(): Buffer | undefined { - const path = env.STACKABLE_COCKPIT_STORAGE_TLS_CA_CERT?.trim(); + const path = storageTlsCaCertPath; if (!path) return undefined; let ca: Buffer; try { diff --git a/src/lib/server/storage/transfer/http.test.ts b/src/lib/server/storage/transfer/http.test.ts index 093dcda1..cfc7319e 100644 --- a/src/lib/server/storage/transfer/http.test.ts +++ b/src/lib/server/storage/transfer/http.test.ts @@ -5,11 +5,7 @@ import type { StorageProvider } from '../provider.js'; import type { TransferRequest } from './resolve.js'; vi.mock('$lib/server/logging', () => import('$lib/test-utils/mock-logger.js')); -vi.mock('$lib/server/feature-flags.js', () => ({ - storagePasteEnabled: true, - storageMoveEnabled: true, - storageRenameEnabled: true -})); +vi.mock('$lib/server/feature-flags.js', () => ({ storageTransferEnabled: true })); const MINUTE = 60_000; diff --git a/src/lib/server/storage/transfer/http.ts b/src/lib/server/storage/transfer/http.ts index fcb075da..c1e73652 100644 --- a/src/lib/server/storage/transfer/http.ts +++ b/src/lib/server/storage/transfer/http.ts @@ -1,10 +1,6 @@ import { error } from '@sveltejs/kit'; import type pino from 'pino'; -import { - storageMoveEnabled, - storagePasteEnabled, - storageRenameEnabled -} from '$lib/server/feature-flags.js'; +import { storageTransferEnabled } from '$lib/server/feature-flags.js'; import { storageTransferDuration, storageTransferItemsTotal } from '$lib/server/metrics.js'; import type { StorageProvider } from '../provider.js'; import { @@ -22,16 +18,9 @@ import { TransferRequestError, validateTransferRequest, type TransferRequest } f /** A running transfer refreshes its job this often, so no copy expires mid-transfer. */ const HEARTBEAT_INTERVAL_MS = 60_000; -/** Whether the feature flags allow `op`: copy needs paste, move needs move or paste, rename needs rename. */ -function transferAllowed(op: TransferOp): boolean { - if (op === 'copy') return storagePasteEnabled; - if (op === 'move') return storageMoveEnabled || storagePasteEnabled; - return storageRenameEnabled; -} - -/** Throws 404 when the feature flags do not allow `op`. */ +/** Throws 404 when transfers (copy, move, rename) are disabled. */ export function requireTransferAllowed(op: TransferOp, logger: pino.Logger, userId?: string): void { - if (transferAllowed(op)) return; + if (storageTransferEnabled) return; logger.warn({ operation: op, user_id: userId }, 'storage operation rejected: feature disabled'); throw error(404, `Storage ${op} is disabled`); } diff --git a/src/lib/storage/bookmarks.svelte.ts b/src/lib/storage/bookmarks.svelte.ts index 7f358c07..3eafd716 100644 --- a/src/lib/storage/bookmarks.svelte.ts +++ b/src/lib/storage/bookmarks.svelte.ts @@ -8,7 +8,7 @@ import { RecentLocationSchema } from './persistence.js'; import { loadFromStorage, persistToStorage } from '$lib/client/local-storage.js'; -import { maxRecentFiles } from '$lib/client/feature-flags.js'; +import { maxRecentFiles } from '$lib/client/config.js'; import { SvelteDate, SvelteSet } from 'svelte/reactivity'; function dedupByKey(items: T[], keyFn: (item: T) => string): T[] { diff --git a/src/lib/storage/clipboard-move-disabled.svelte.spec.ts b/src/lib/storage/clipboard-transfer-disabled.svelte.spec.ts similarity index 65% rename from src/lib/storage/clipboard-move-disabled.svelte.spec.ts rename to src/lib/storage/clipboard-transfer-disabled.svelte.spec.ts index ea5f908a..00f1c2a6 100644 --- a/src/lib/storage/clipboard-move-disabled.svelte.spec.ts +++ b/src/lib/storage/clipboard-transfer-disabled.svelte.spec.ts @@ -1,17 +1,6 @@ -// The flags are read once per module in the browser project, so the disabled -// move flag gets its own spec file. -vi.mock('$lib/client/feature-flags.js', () => ({ - storageRestoreTabsEnabled: false, - allowedPageSizes: [25, 50, 100], - defaultPageSize: 25, - maxRecentFiles: 15, - maxEditableFileSize: 5 * 1024 * 1024, - uploadConcurrency: 3, - storageCutCopyEnabled: true, - storagePasteEnabled: true, - storageRenameEnabled: true, - storageMoveEnabled: false -})); +// The flags are read once per module in the browser project, so disabled +// transfers get their own spec file. +vi.mock('$lib/client/feature-flags.js', () => ({ storageTransferEnabled: false })); vi.mock('$lib/storage/connection-store.svelte.js', () => ({ connectionStore: { activeConnectionId: 'test-connection-id' } @@ -29,7 +18,7 @@ import { describe, it, expect, vi } from 'vitest'; import { StorageState } from './state.svelte.js'; import { createMemoryStorageApi } from './api.test-utils.js'; -describe('drag and drop with moving disabled', () => { +describe('drag and drop with transfers disabled', () => { it('ignores drops', () => { const transfer = vi.fn(); const state = new StorageState({ diff --git a/src/lib/storage/clipboard.svelte.spec.ts b/src/lib/storage/clipboard.svelte.spec.ts index e8451d1a..4311fdb0 100644 --- a/src/lib/storage/clipboard.svelte.spec.ts +++ b/src/lib/storage/clipboard.svelte.spec.ts @@ -1,15 +1,4 @@ -vi.mock('$lib/client/feature-flags.js', () => ({ - storageMoveEnabled: true, - storageRestoreTabsEnabled: false, - allowedPageSizes: [25, 50, 100], - defaultPageSize: 25, - maxRecentFiles: 15, - maxEditableFileSize: 5 * 1024 * 1024, - uploadConcurrency: 3, - storageCutCopyEnabled: true, - storagePasteEnabled: true, - storageRenameEnabled: true -})); +vi.mock('$lib/client/feature-flags.js', () => ({ storageTransferEnabled: true })); vi.mock('$lib/storage/connection-store.svelte.js', () => ({ connectionStore: { activeConnectionId: 'test-connection-id' } diff --git a/src/lib/storage/clipboard.svelte.ts b/src/lib/storage/clipboard.svelte.ts index af374884..e2d73a10 100644 --- a/src/lib/storage/clipboard.svelte.ts +++ b/src/lib/storage/clipboard.svelte.ts @@ -6,7 +6,7 @@ import type { ConflictEntry } from '$lib/components/storage/modals/shared/confli import { keyToName, parentPrefix } from '$lib/storage/utils.js'; import { StorageError, getActionErrorMessage } from './errors.js'; import { pageUnloading } from './operations.svelte.js'; -import { storageMoveEnabled } from '$lib/client/feature-flags.js'; +import { storageTransferEnabled } from '$lib/client/feature-flags.js'; import { OperationsState } from './operations.svelte.js'; import type { StorageApi } from './api.js'; import { MAX_TRANSFER_ITEMS, type PlanItem, type TransferItemRequest } from './transfer-types.js'; @@ -111,14 +111,14 @@ export class ClipboardState { ); } - /** Set clipboard with cut action. Caller should check storageCutCopyEnabled before calling. */ + /** Set clipboard with cut action. Caller should check storageTransferEnabled before calling. */ cut(selectedKeys: string[], objects: StoragePage, bucket: string, prefix: string): void { if (selectedKeys.length === 0) return; this.clipboard = this._clipboardFor('cut', selectedKeys, objects, bucket, prefix); addToast('info', m.storage_action_cut_success({ count: selectedKeys.length })); } - /** Set clipboard with copy action. Caller should check storageCutCopyEnabled before calling. */ + /** Set clipboard with copy action. Caller should check storageTransferEnabled before calling. */ copy(selectedKeys: string[], objects: StoragePage, bucket: string, prefix: string): void { if (selectedKeys.length === 0) return; this.clipboard = this._clipboardFor('copy', selectedKeys, objects, bucket, prefix); @@ -129,7 +129,7 @@ export class ClipboardState { /** * Paste the clipboard into `destPrefix` of the current bucket. Caller should - * check storagePasteEnabled and that the clipboard is non-empty first. + * check storageTransferEnabled and that the clipboard is non-empty first. */ async paste(destPrefix: string): Promise { const clipboard = this.clipboard; @@ -180,7 +180,7 @@ export class ClipboardState { bucket: this._callbacks.getBucket() } ): void => { - if (!storageMoveEnabled) return; + if (!storageTransferEnabled) return; if (source.connectionId !== this._callbacks.getConnectionId()) { addToast('warning', m.storage_transfer_other_connection()); return; diff --git a/src/lib/storage/drag-handlers.ts b/src/lib/storage/drag-handlers.ts index 5c608996..5e97bc37 100644 --- a/src/lib/storage/drag-handlers.ts +++ b/src/lib/storage/drag-handlers.ts @@ -1,5 +1,5 @@ import { SvelteSet } from 'svelte/reactivity'; -import { storageCutCopyEnabled, storageMoveEnabled } from '$lib/client/feature-flags.js'; +import { storageTransferEnabled } from '$lib/client/feature-flags.js'; import { connectionStore } from './connection-store.svelte.js'; import type { DragSource } from './clipboard.svelte.js'; @@ -19,7 +19,7 @@ export function handleRowDragStart( itemKey: string, state: { selectedKeys: Set; bucket: string; archive: { isInArchive: boolean } } ): void { - if (!storageCutCopyEnabled || state.archive.isInArchive) return; + if (!storageTransferEnabled || state.archive.isInArchive) return; if (!state.selectedKeys.has(itemKey)) { state.selectedKeys = new SvelteSet([itemKey]); } @@ -55,5 +55,5 @@ export function parseStorageDrop(e: DragEvent): { keys: string[]; source: DragSo * Check whether a storage drag-drop is allowed in the current context. */ export function canStorageDrop(storage: { archive: { isInArchive: boolean } }): boolean { - return storageMoveEnabled && !storage.archive.isInArchive; + return storageTransferEnabled && !storage.archive.isInArchive; } diff --git a/src/lib/storage/keyboard-scope.svelte.spec.ts b/src/lib/storage/keyboard-scope.svelte.spec.ts index fde7af27..7c91f52b 100644 --- a/src/lib/storage/keyboard-scope.svelte.spec.ts +++ b/src/lib/storage/keyboard-scope.svelte.spec.ts @@ -1,15 +1,4 @@ -vi.mock('$lib/client/feature-flags.js', () => ({ - storageRestoreTabsEnabled: false, - allowedPageSizes: [25, 50, 100], - defaultPageSize: 25, - maxRecentFiles: 15, - maxEditableFileSize: 5 * 1024 * 1024, - uploadConcurrency: 3, - storageCutCopyEnabled: true, - storagePasteEnabled: true, - storageRenameEnabled: true, - storageMoveEnabled: true -})); +vi.mock('$lib/client/feature-flags.js', () => ({ storageTransferEnabled: true })); vi.mock('$lib/storage/connection-store.svelte.js', () => ({ connectionStore: { activeConnectionId: 'test-connection-id' } diff --git a/src/lib/storage/pagination-history.svelte.spec.ts b/src/lib/storage/pagination-history.svelte.spec.ts index a3b6d705..55a8fe3b 100644 --- a/src/lib/storage/pagination-history.svelte.spec.ts +++ b/src/lib/storage/pagination-history.svelte.spec.ts @@ -1,15 +1,4 @@ -vi.mock('$lib/client/feature-flags.js', () => ({ - storageRestoreTabsEnabled: false, - allowedPageSizes: [25, 50, 100], - defaultPageSize: 25, - maxRecentFiles: 15, - maxEditableFileSize: 5 * 1024 * 1024, - uploadConcurrency: 3, - storageCutCopyEnabled: true, - storagePasteEnabled: true, - storageRenameEnabled: true, - storageMoveEnabled: true -})); +vi.mock('$lib/client/feature-flags.js', () => ({ storageTransferEnabled: true })); vi.mock('$lib/storage/connection-store.svelte.js', () => ({ connectionStore: { activeConnectionId: 'test-connection-id' } diff --git a/src/lib/storage/state.svelte.spec.ts b/src/lib/storage/state.svelte.spec.ts index da10755e..cfa15917 100644 --- a/src/lib/storage/state.svelte.spec.ts +++ b/src/lib/storage/state.svelte.spec.ts @@ -1,15 +1,4 @@ -vi.mock('$lib/client/feature-flags.js', () => ({ - storageRestoreTabsEnabled: false, - allowedPageSizes: [25, 50, 100], - defaultPageSize: 25, - maxRecentFiles: 15, - maxEditableFileSize: 5 * 1024 * 1024, - uploadConcurrency: 3, - storageCutCopyEnabled: true, - storagePasteEnabled: true, - storageRenameEnabled: true, - storageMoveEnabled: true -})); +vi.mock('$lib/client/feature-flags.js', () => ({ storageTransferEnabled: true })); vi.mock('$lib/storage/connection-store.svelte.js', () => ({ connectionStore: { activeConnectionId: 'test-connection-id' } diff --git a/src/lib/storage/state.svelte.ts b/src/lib/storage/state.svelte.ts index 15399700..d5f350db 100644 --- a/src/lib/storage/state.svelte.ts +++ b/src/lib/storage/state.svelte.ts @@ -12,12 +12,8 @@ import type { StorageOperation } from '$lib/storage/types.js'; import { initPageSize, type PageSize } from '$lib/types/pagination.js'; -import { - defaultPageSize, - storageCutCopyEnabled, - storagePasteEnabled, - storageRenameEnabled -} from '$lib/client/feature-flags.js'; +import { defaultPageSize } from '$lib/client/config.js'; +import { storageTransferEnabled } from '$lib/client/feature-flags.js'; import { estimateArchiveSize, startDownload, @@ -645,7 +641,7 @@ export class StorageState { } case 'cut': { - if (!storageCutCopyEnabled) return; + if (!storageTransferEnabled) return; const cutKeys = [...this.selectedKeys]; if (cutKeys.length === 0) return; this.clipboardState.cut(cutKeys, this.objects, this.bucket, this.prefix); @@ -653,7 +649,7 @@ export class StorageState { } case 'copy': { - if (!storageCutCopyEnabled) return; + if (!storageTransferEnabled) return; const copyKeys = [...this.selectedKeys]; if (copyKeys.length === 0) return; this.clipboardState.copy(copyKeys, this.objects, this.bucket, this.prefix); @@ -661,7 +657,7 @@ export class StorageState { } case 'paste': { - if (!storagePasteEnabled) return; + if (!storageTransferEnabled) return; if (!this.clipboardState.clipboard || this.clipboardState.clipboard.keys.length === 0) return; if (!this.clipboardState.canPaste) { @@ -679,7 +675,7 @@ export class StorageState { } case 'rename': { - if (!storageRenameEnabled) return; + if (!storageTransferEnabled) return; const renameKey = ctxKey ?? [...this.selectedKeys][0]; if (!renameKey) return; this.renameError = null; diff --git a/src/lib/storage/tabs.svelte.spec.ts b/src/lib/storage/tabs.svelte.spec.ts index 315ca951..2eef8078 100644 --- a/src/lib/storage/tabs.svelte.spec.ts +++ b/src/lib/storage/tabs.svelte.spec.ts @@ -15,14 +15,10 @@ function makeStorage(bucket = 'test-bucket', prefix = ''): StorageState { return state; } -function makeTabs( - storage: StorageState, - opts?: { persistEnabled?: boolean; connectionId?: string | null } -) { +function makeTabs(storage: StorageState, opts?: { connectionId?: string | null }) { const navigateToLocation = vi.fn(); const replaceLocationUrl = vi.fn(); const ts = new TabsState(storage, { - persistEnabled: opts?.persistEnabled ?? false, connectionId: opts?.connectionId ?? null, navigateToLocation, replaceLocationUrl @@ -394,23 +390,16 @@ describe('TabsState', () => { // ── persistence ─────────────────────────────────────────────────────────── describe('peekPersistedTabs', () => { - it('returns null when persistence is disabled', () => { - const storage = makeStorage(); - const { ts } = makeTabs(storage, { persistEnabled: false }); - - expect(ts.peekPersistedTabs()).toBeNull(); - }); - it('returns null when nothing is saved in localStorage', () => { const storage = makeStorage(); - const { ts } = makeTabs(storage, { persistEnabled: true }); + const { ts } = makeTabs(storage); expect(ts.peekPersistedTabs()).toBeNull(); }); it('returns saved data after ensureInitialTab saves it', () => { const storage = makeStorage('bucket', ''); - const { ts } = makeTabs(storage, { persistEnabled: true }); + const { ts } = makeTabs(storage); ts.ensureInitialTab(); const peeked = ts.peekPersistedTabs(); @@ -428,7 +417,7 @@ describe('TabsState', () => { localStorage.setItem(LS_TABS, JSON.stringify(saved)); const storage = makeStorage(); - const { ts } = makeTabs(storage, { persistEnabled: true, connectionId: 'conn-b' }); + const { ts } = makeTabs(storage, { connectionId: 'conn-b' }); expect(ts.peekPersistedTabs()).toBeNull(); }); @@ -442,7 +431,7 @@ describe('TabsState', () => { localStorage.setItem(LS_TABS, JSON.stringify(saved)); const storage = makeStorage(); - const { ts } = makeTabs(storage, { persistEnabled: true, connectionId: 'conn-a' }); + const { ts } = makeTabs(storage, { connectionId: 'conn-a' }); expect(ts.peekPersistedTabs()).toBeNull(); }); @@ -456,7 +445,7 @@ describe('TabsState', () => { localStorage.setItem(LS_TABS, JSON.stringify(saved)); const storage = makeStorage(); - const { ts } = makeTabs(storage, { persistEnabled: true, connectionId: 'conn-a' }); + const { ts } = makeTabs(storage, { connectionId: 'conn-a' }); expect(ts.peekPersistedTabs()).not.toBeNull(); }); @@ -465,7 +454,7 @@ describe('TabsState', () => { describe('clearPersistedTabs', () => { it('removes the entry from localStorage', () => { const storage = makeStorage(); - const { ts } = makeTabs(storage, { persistEnabled: true }); + const { ts } = makeTabs(storage); ts.ensureInitialTab(); ts.clearPersistedTabs(); @@ -475,9 +464,9 @@ describe('TabsState', () => { }); describe('ensureInitialTab persistence write', () => { - it('writes tab data to localStorage when persistEnabled', () => { + it('writes tab data to localStorage', () => { const storage = makeStorage('bucket', ''); - const { ts } = makeTabs(storage, { persistEnabled: true }); + const { ts } = makeTabs(storage); ts.ensureInitialTab(); const raw = localStorage.getItem(LS_TABS); @@ -580,14 +569,14 @@ describe('TabsState', () => { localStorage.setItem(LS_TABS, JSON.stringify(saved)); const storage = makeStorage('b1', ''); - const { ts } = makeTabs(storage, { persistEnabled: true, connectionId: 'conn-a' }); + const { ts } = makeTabs(storage, { connectionId: 'conn-a' }); ts.ensureInitialTab(); // Should have restored 2 tabs rather than creating 1 fresh tab expect(ts.tabs.length).toBe(2); }); - it('restores from localStorage on every ensureInitialTab when persistEnabled', () => { + it('restores from localStorage on every ensureInitialTab', () => { const saved: PersistedTabsState = { tabs: [ { id: 'a', label: 'A', connection: 's3.example.com', bucket: 'b1', prefix: '' }, @@ -600,14 +589,14 @@ describe('TabsState', () => { // First TabsState restores from localStorage const storage1 = makeStorage('b1', ''); - const { ts: ts1 } = makeTabs(storage1, { persistEnabled: true, connectionId: 'conn-a' }); + const { ts: ts1 } = makeTabs(storage1, { connectionId: 'conn-a' }); ts1.ensureInitialTab(); expect(ts1.tabs.length).toBe(2); // Second TabsState — persistence is still enabled and data exists, // so it restores from localStorage again (handles page reloads) const storage2 = makeStorage('b1', ''); - const { ts: ts2 } = makeTabs(storage2, { persistEnabled: true, connectionId: 'conn-a' }); + const { ts: ts2 } = makeTabs(storage2, { connectionId: 'conn-a' }); ts2.ensureInitialTab(); expect(ts2.tabs.length).toBe(2); }); @@ -618,7 +607,7 @@ describe('TabsState', () => { describe('removeDeletedDirectories', () => { it('closes inactive tabs inside a deleted folder and forgets them', () => { const storage = makeStorage('bucket', 'archive/2025/'); - const { ts } = makeTabs(storage, { persistEnabled: true, connectionId: 'conn-1' }); + const { ts } = makeTabs(storage, { connectionId: 'conn-1' }); ts.ensureInitialTab(); ts.addTab(); storage.prefix = ''; diff --git a/src/lib/storage/tabs.svelte.ts b/src/lib/storage/tabs.svelte.ts index 1cb8ec82..6dc364b7 100644 --- a/src/lib/storage/tabs.svelte.ts +++ b/src/lib/storage/tabs.svelte.ts @@ -72,7 +72,6 @@ export class TabsState { bucket: string; prefix: string; } | null = null; - private persistEnabled: boolean; private connectionId: string | null; private navigateToLocation: ((connection: string, bucket: string, prefix: string) => void) | null; private replaceLocationUrl: ((connection: string, bucket: string, prefix: string) => void) | null; @@ -84,14 +83,12 @@ export class TabsState { constructor( storage: StorageState, options?: { - persistEnabled?: boolean; connectionId?: string | null; navigateToLocation?: (connection: string, bucket: string, prefix: string) => void; replaceLocationUrl?: (connection: string, bucket: string, prefix: string) => void; } ) { this.storage = storage; - this.persistEnabled = options?.persistEnabled ?? false; this.connectionId = options?.connectionId ?? null; this.navigateToLocation = options?.navigateToLocation ?? null; this.replaceLocationUrl = options?.replaceLocationUrl ?? null; @@ -158,7 +155,7 @@ export class TabsState { // ── Persistence ────────────────────────────────────────────────────────── private saveToPersistence(): void { - if (!this.persistEnabled || !browser) return; + if (!browser) return; const data: PersistedTabsState = { tabs: this.tabs.map((t) => ({ id: t.id, @@ -174,10 +171,10 @@ export class TabsState { } /** Reads persisted tab state from localStorage without modifying any state. - * Returns null if nothing is saved, persistence is disabled, or the saved + * Returns null if nothing is saved or the saved * state belongs to a different connection. */ peekPersistedTabs(): PersistedTabsState | null { - if (!this.persistEnabled || !browser) return null; + if (!browser) return null; try { const raw = localStorage.getItem(LS_TABS); if (!raw) return null; @@ -256,9 +253,9 @@ export class TabsState { // ── Tab operations ─────────────────────────────────────────────────────── - /** Initialises tabs from the current storage state. If persistence is - * enabled, restores tabs from localStorage (handles both SPA navigation - * via the /storage banner flag and full page reloads). + /** Initialises tabs from the current storage state, restoring them from + * localStorage (handles both SPA navigation via the /storage banner flag + * and full page reloads). * * Tabs are only restored when the current storage bucket/prefix matches * the saved active tab's location — otherwise the user navigated to a @@ -266,22 +263,20 @@ export class TabsState { ensureInitialTab(): void { if (this.tabs.length > 0) return; - if (this.persistEnabled) { - const saved = this.peekPersistedTabs(); - if (saved && saved.tabs.length > 0) { - const activeIdx = saved.tabs.findIndex((pt) => pt.id === saved.activeTabId); - const activeTab = activeIdx >= 0 ? saved.tabs[activeIdx] : saved.tabs[0]; - if ( - activeTab && - this.storage.connectionHostname === activeTab.connection && - this.storage.bucket === activeTab.bucket && - this.storage.prefix === activeTab.prefix - ) { - this.restorePersistedTabs(saved); - return; - } - this.clearPersistedTabs(); + const saved = this.peekPersistedTabs(); + if (saved && saved.tabs.length > 0) { + const activeIdx = saved.tabs.findIndex((pt) => pt.id === saved.activeTabId); + const activeTab = activeIdx >= 0 ? saved.tabs[activeIdx] : saved.tabs[0]; + if ( + activeTab && + this.storage.connectionHostname === activeTab.connection && + this.storage.bucket === activeTab.bucket && + this.storage.prefix === activeTab.prefix + ) { + this.restorePersistedTabs(saved); + return; } + this.clearPersistedTabs(); } const id = crypto.randomUUID(); diff --git a/src/lib/types/pagination.ts b/src/lib/types/pagination.ts index f739217c..f1e8cf89 100644 --- a/src/lib/types/pagination.ts +++ b/src/lib/types/pagination.ts @@ -1,5 +1,5 @@ import { browser } from '$app/environment'; -import { allowedPageSizes, defaultPageSize } from '$lib/client/feature-flags.js'; +import { allowedPageSizes, defaultPageSize } from '$lib/client/config.js'; export type PageSize = number; diff --git a/src/routes/(app)/api/storage/archive/extract/+server.ts b/src/routes/(app)/api/storage/archive/extract/+server.ts index 268a190b..fde073f3 100644 --- a/src/routes/(app)/api/storage/archive/extract/+server.ts +++ b/src/routes/(app)/api/storage/archive/extract/+server.ts @@ -7,7 +7,7 @@ import { import { getProvider } from '$lib/server/storage/utils.js'; import { requireObjectKey } from '$lib/server/storage/request-context.js'; import { withStorageHttpErrors } from '$lib/server/storage/wrap-provider.js'; -import { archivePreviewMaxBytes } from '$lib/server/feature-flags.js'; +import { archivePreviewMaxBytes } from '$lib/server/storage/config.js'; import type { RequestHandler } from './$types'; /** diff --git a/src/routes/(app)/api/storage/archive/listing/+server.ts b/src/routes/(app)/api/storage/archive/listing/+server.ts index 2e3088dd..2bea994a 100644 --- a/src/routes/(app)/api/storage/archive/listing/+server.ts +++ b/src/routes/(app)/api/storage/archive/listing/+server.ts @@ -3,7 +3,7 @@ import { ArchiveCacheFullError, listArchiveContents } from '$lib/server/storage/ import { getProvider } from '$lib/server/storage/utils.js'; import { requireObjectKey } from '$lib/server/storage/request-context.js'; import { withStorageHttpErrors } from '$lib/server/storage/wrap-provider.js'; -import { archivePreviewMaxBytes } from '$lib/server/feature-flags.js'; +import { archivePreviewMaxBytes } from '$lib/server/storage/config.js'; import type { RequestHandler } from './$types'; /** diff --git a/src/routes/(app)/api/storage/preview/+server.ts b/src/routes/(app)/api/storage/preview/+server.ts index b233f1db..9c3226bf 100644 --- a/src/routes/(app)/api/storage/preview/+server.ts +++ b/src/routes/(app)/api/storage/preview/+server.ts @@ -7,13 +7,13 @@ import { getCsvPreview } from '$lib/server/storage/preview/csv'; import { binaryPreview, KNOWN_BINARY_TYPES } from '$lib/server/storage/preview/binary.js'; import { streamPreview } from '$lib/server/storage/preview/stream.js'; import { createStorageProvider, requireObjectKey } from '$lib/server/storage/request-context.js'; -import { infiniteScrollEnabled, filePreviewRows } from '$lib/server/feature-flags'; +import { filePreviewRows } from '$lib/server/storage/config.js'; import { PREVIEW_CHUNK_ROWS } from '$lib/storage/types.js'; import type { RequestHandler } from './$types'; -/** Upper bound for rows returned by a single preview request. Applies whether - * or not infinite scroll is enabled, so one request can never ask the server - * to decode an unbounded number of rows. Never below the client chunk size. */ +/** Upper bound for rows returned by a single preview request, so one request + * can never ask the server to decode an unbounded number of rows. Never below + * the client chunk size. */ const maxPreviewRowsPerRequest = Math.max(filePreviewRows, PREVIEW_CHUNK_ROWS); const previewRangeSchema = z.object({ @@ -58,7 +58,7 @@ export const GET: RequestHandler = async (event) => { `offset must be a non-negative integer and limit an integer between 1 and ${maxPreviewRowsPerRequest}` ); } - let { offset, limit } = range.data; + const { offset, limit } = range.data; try { const metadata = await provider.getMetadata(key); @@ -74,12 +74,6 @@ export const GET: RequestHandler = async (event) => { const includeData = url.searchParams.get('data') === 'true'; - // When infinite scroll is disabled, restrict to the first page only (no chunked loading) - if (!infiniteScrollEnabled) { - offset = 0; - limit = Math.min(limit, filePreviewRows); - } - if (isParquet) { return await getParquetPreview( provider, diff --git a/src/routes/(app)/api/storage/preview/server.test.ts b/src/routes/(app)/api/storage/preview/server.test.ts index 6a2c1984..4b50cfec 100644 --- a/src/routes/(app)/api/storage/preview/server.test.ts +++ b/src/routes/(app)/api/storage/preview/server.test.ts @@ -19,21 +19,14 @@ vi.mock('$lib/server/storage/preview/stream.js', () => ({ streamPreview: vi.fn(async () => new Response('preview content')) })); -const flags = vi.hoisted(() => ({ infiniteScrollEnabled: false })); - -vi.mock('$lib/server/feature-flags', () => ({ - get infiniteScrollEnabled() { - return flags.infiniteScrollEnabled; - }, +vi.mock('$lib/server/storage/config.js', () => ({ filePreviewRows: 250, textPreviewBytes: 262144, imagePreviewBytes: 5242880, pdfPreviewBytes: 26214400, - archivePreviewMaxBytes: 104857600, + filePreviewColumns: 50, maxEditableFileSize: 5242880, - parquetDisallowedCompression: [], - completionEnabled: true, - storageBrowserEnabled: true + parquetDisallowedCompression: [] })); vi.mock('$lib/server/storage/preview/csv.js', () => ({ @@ -89,7 +82,6 @@ function mockEvent(params: string) { describe('GET /api/storage/preview', () => { beforeEach(() => { vi.clearAllMocks(); - flags.infiniteScrollEnabled = false; }); it('returns binary preview for known binary types', async () => { @@ -243,27 +235,6 @@ describe('GET /api/storage/preview', () => { expect(res.headers.get('X-Preview-Format')).toBe('parquet'); }); - it('resets offset and caps limit when infinite scroll is disabled', async () => { - mockGetMetadata.mockResolvedValue({ - contentType: 'application/x-parquet', - size: 50000 - }); - - await GET(mockEvent('bucket=b1&key=large.parquet&offset=500&limit=100')); - - expect(getParquetPreview).toHaveBeenCalledWith( - expect.anything(), - 'large.parquet', - 0, - 100, - expect.anything(), - expect.objectContaining({ size: 50000 }), - false, - 'b1', - 'connection-1' - ); - }); - it('handles zero-size parquet file gracefully', async () => { mockGetMetadata.mockResolvedValue({ contentType: 'application/vnd.apache.parquet', @@ -315,8 +286,7 @@ describe('GET /api/storage/preview', () => { ['offset=1.5'], ['offset=abc'], ['offset=1e400'] - ])('rejects %s with 400 when infinite scroll is enabled', async (params) => { - flags.infiniteScrollEnabled = true; + ])('rejects %s with 400', async (params) => { mockGetMetadata.mockResolvedValue({ contentType: 'application/x-parquet', size: 50000 }); await expect(GET(mockEvent(`bucket=b1&key=large.parquet&${params}`))).rejects.toMatchObject({ @@ -326,17 +296,7 @@ describe('GET /api/storage/preview', () => { expect(getParquetPreview).not.toHaveBeenCalled(); }); - it('rejects an oversized limit with 400 when infinite scroll is disabled', async () => { - mockGetMetadata.mockResolvedValue({ contentType: 'text/csv', size: 50000 }); - - await expect( - GET(mockEvent('bucket=b1&key=large.csv&limit=1000000000')) - ).rejects.toMatchObject({ status: 400 }); - expect(getCsvPreview).not.toHaveBeenCalled(); - }); - - it('passes in-range chunks through when infinite scroll is enabled', async () => { - flags.infiniteScrollEnabled = true; + it('passes in-range chunks through', async () => { mockGetMetadata.mockResolvedValue({ contentType: 'application/x-parquet', size: 50000 }); await GET(mockEvent('bucket=b1&key=large.parquet&offset=1000000&limit=250&data=true')); @@ -353,15 +313,5 @@ describe('GET /api/storage/preview', () => { 'connection-1' ); }); - - it('bounds CSV chunk requests when infinite scroll is enabled', async () => { - flags.infiniteScrollEnabled = true; - mockGetMetadata.mockResolvedValue({ contentType: 'text/csv', size: 100 }); - - await expect( - GET(mockEvent('bucket=b1&key=data.csv&offset=0&limit=1000000000&data=true')) - ).rejects.toMatchObject({ status: 400 }); - expect(getCsvPreview).not.toHaveBeenCalled(); - }); }); }); diff --git a/src/routes/(app)/api/storage/save-text/+server.ts b/src/routes/(app)/api/storage/save-text/+server.ts index 08e9e405..02afb80f 100644 --- a/src/routes/(app)/api/storage/save-text/+server.ts +++ b/src/routes/(app)/api/storage/save-text/+server.ts @@ -1,7 +1,7 @@ import { error, isHttpError, json } from '@sveltejs/kit'; import { z } from 'zod'; import { createStorageProvider, requireObjectKey } from '$lib/server/storage/request-context.js'; -import { maxEditableFileSize } from '$lib/server/feature-flags.js'; +import { maxEditableFileSize } from '$lib/server/storage/config.js'; import type { RequestHandler } from './$types'; const saveTextQuerySchema = z.object({ diff --git a/src/routes/(app)/api/storage/save-text/server.test.ts b/src/routes/(app)/api/storage/save-text/server.test.ts index 0ae19e26..a31004a7 100644 --- a/src/routes/(app)/api/storage/save-text/server.test.ts +++ b/src/routes/(app)/api/storage/save-text/server.test.ts @@ -11,7 +11,7 @@ vi.mock('$lib/server/storage/utils.js', () => ({ getProvider: () => mockProvider })); -vi.mock('$lib/server/feature-flags.js', () => ({ +vi.mock('$lib/server/storage/config.js', () => ({ maxEditableFileSize: 5242880 })); diff --git a/src/routes/(app)/api/storage/transfer/+server.ts b/src/routes/(app)/api/storage/transfer/+server.ts index e622c504..bca57cf2 100644 --- a/src/routes/(app)/api/storage/transfer/+server.ts +++ b/src/routes/(app)/api/storage/transfer/+server.ts @@ -1,7 +1,7 @@ import type { RequestHandler } from './$types'; import { error } from '@sveltejs/kit'; import { createStorageProviderForBucket } from '$lib/server/storage/request-context.js'; -import { storageTransferConcurrency } from '$lib/server/feature-flags.js'; +import { storageTransferConcurrency } from '$lib/server/storage/config.js'; import { requireTransferAllowed, requireValidTransfer, diff --git a/src/routes/(app)/api/storage/transfer/plan/+server.ts b/src/routes/(app)/api/storage/transfer/plan/+server.ts index 4ddba3bf..e0e37cee 100644 --- a/src/routes/(app)/api/storage/transfer/plan/+server.ts +++ b/src/routes/(app)/api/storage/transfer/plan/+server.ts @@ -1,7 +1,7 @@ import type { RequestHandler } from './$types'; import { error, json } from '@sveltejs/kit'; import { createStorageProviderForBucket } from '$lib/server/storage/request-context.js'; -import { storageTransferConcurrency } from '$lib/server/feature-flags.js'; +import { storageTransferConcurrency } from '$lib/server/storage/config.js'; import { requireTransferAllowed, requireValidTransfer } from '$lib/server/storage/transfer/http.js'; import { resolveItems, diff --git a/src/routes/(app)/api/storage/transfer/plan/server.test.ts b/src/routes/(app)/api/storage/transfer/plan/server.test.ts index bee72418..04140ea6 100644 --- a/src/routes/(app)/api/storage/transfer/plan/server.test.ts +++ b/src/routes/(app)/api/storage/transfer/plan/server.test.ts @@ -8,22 +8,18 @@ import { import type { StorageProvider } from '$lib/server/storage/provider.js'; const { flags, state } = vi.hoisted(() => ({ - flags: { paste: true, move: true }, + flags: { transfer: true }, state: { store: null as unknown as MemoryStore, wrap: (provider: StorageProvider) => provider } })); vi.mock('$lib/server/feature-flags.js', () => ({ - get storagePasteEnabled() { - return flags.paste; - }, - get storageMoveEnabled() { - return flags.move; - }, - storageRenameEnabled: true, - storageTransferConcurrency: 2 + get storageTransferEnabled() { + return flags.transfer; + } })); +vi.mock('$lib/server/storage/config.js', () => ({ storageTransferConcurrency: 2 })); vi.mock('$lib/server/storage/request-context.js', () => ({ createStorageProviderForBucket: (_event: unknown, bucket: string) => state.wrap(memoryProvider(state.store, bucket)) @@ -50,7 +46,7 @@ const plan = (keys: string[], prefix = 'dest/', op = 'copy') => ({ describe('POST /api/storage/transfer/plan', () => { beforeEach(() => { - Object.assign(flags, { paste: true, move: true }); + flags.transfer = true; state.store = memoryStore({ b: ['a.txt', 'docs/', 'dest/a.txt', 'dest/docs/x'] }); state.wrap = (provider) => provider; }); @@ -131,15 +127,15 @@ describe('POST /api/storage/transfer/plan', () => { ).rejects.toMatchObject({ status: 400 }); }); - it.each([ - ['copy', { paste: false, move: true }], - ['move', { paste: false, move: false }] - ])('answers 404 when the previewed %s is disabled', async (op, flagState) => { - Object.assign(flags, flagState); - await expect(POST(event(plan(['a.txt'], 'dest/', op)))).rejects.toMatchObject({ - status: 404 - }); - }); + it.each(['copy', 'move'])( + 'answers 404 for a previewed %s when transfers are disabled', + async (op) => { + flags.transfer = false; + await expect(POST(event(plan(['a.txt'], 'dest/', op)))).rejects.toMatchObject({ + status: 404 + }); + } + ); it('passes storage errors through as HTTP errors', async () => { state.wrap = (provider) => ({ diff --git a/src/routes/(app)/api/storage/transfer/server.test.ts b/src/routes/(app)/api/storage/transfer/server.test.ts index f5a403db..1edcd3fc 100644 --- a/src/routes/(app)/api/storage/transfer/server.test.ts +++ b/src/routes/(app)/api/storage/transfer/server.test.ts @@ -8,22 +8,16 @@ import { } from '$lib/server/storage/transfer/memory-provider.test-utils.js'; const { flags, state } = vi.hoisted(() => ({ - flags: { paste: true, move: true, rename: true }, + flags: { transfer: true }, state: { store: null as unknown as MemoryStore } })); vi.mock('$lib/server/logging', () => import('$lib/test-utils/mock-logger.js')); vi.mock('$lib/server/feature-flags.js', () => ({ - get storagePasteEnabled() { - return flags.paste; - }, - get storageMoveEnabled() { - return flags.move; - }, - get storageRenameEnabled() { - return flags.rename; - }, - storageTransferConcurrency: 2 + get storageTransferEnabled() { + return flags.transfer; + } })); +vi.mock('$lib/server/storage/config.js', () => ({ storageTransferConcurrency: 2 })); vi.mock('$lib/server/storage/request-context.js', () => ({ createStorageProviderForBucket: (_event: unknown, bucket: string) => memoryProvider(state.store, bucket) @@ -70,7 +64,7 @@ function copy(keys: string[], extra: Record = {}) { describe('POST /api/storage/transfer', () => { beforeEach(() => { - Object.assign(flags, { paste: true, move: true, rename: true }); + flags.transfer = true; state.store = memoryStore({ b: ['a.txt', 'docs/x.txt'], other: [] }); }); @@ -147,24 +141,17 @@ describe('POST /api/storage/transfer', () => { await expect(POST(event(body))).rejects.toMatchObject({ status: 409 }); }); - it.each([ - ['copy', { paste: false, move: true, rename: true }], - ['move', { paste: false, move: false, rename: true }], - ['rename', { paste: true, move: true, rename: false }] - ])('answers 404 for %s when its flag is off', async (op, flagState) => { - Object.assign(flags, flagState); - const body = - op === 'rename' - ? { ...copy([]), op, dest: undefined, items: [{ sourceKey: 'a.txt', name: 'b.txt' }] } - : { ...copy(['a.txt']), op }; - await expect(POST(event(body))).rejects.toMatchObject({ status: 404 }); - }); - - it('allows a move with only the paste flag', async () => { - Object.assign(flags, { paste: true, move: false, rename: false }); - const response = await POST(event({ ...copy(['a.txt']), op: 'move' })); - expect((await lines(response)).at(-1).type).toBe('complete'); - }); + it.each(['copy', 'move', 'rename'])( + 'answers 404 for %s when transfers are disabled', + async (op) => { + flags.transfer = false; + const body = + op === 'rename' + ? { ...copy([]), op, dest: undefined, items: [{ sourceKey: 'a.txt', name: 'b.txt' }] } + : { ...copy(['a.txt']), op }; + await expect(POST(event(body))).rejects.toMatchObject({ status: 404 }); + } + ); it('keeps every original when a running move is cancelled', async () => { state.store = memoryStore({ b: Array.from({ length: 20 }, (_, i) => `many/${i}.txt`) }); diff --git a/src/routes/(app)/storage/+layout.svelte b/src/routes/(app)/storage/+layout.svelte index ef0849c3..b009fbd8 100644 --- a/src/routes/(app)/storage/+layout.svelte +++ b/src/routes/(app)/storage/+layout.svelte @@ -10,7 +10,6 @@ import { connectionHostname, connectionStore } from '$lib/storage/connection-store.svelte.js'; import { createFetchStorageApi } from '$lib/storage/api.js'; import { TabsState } from '$lib/storage/tabs.svelte.js'; - import { storageRestoreTabsEnabled } from '$lib/client/feature-flags.js'; let { children, data } = $props(); @@ -36,7 +35,6 @@ } const tabsState = new TabsState(storage, { - persistEnabled: storageRestoreTabsEnabled, connectionId: storage.connectionId, navigateToLocation: (connection, bucket, prefix) => { void goto( diff --git a/src/routes/(app)/storage/+page.svelte b/src/routes/(app)/storage/+page.svelte index ad87eb38..9de29cf8 100644 --- a/src/routes/(app)/storage/+page.svelte +++ b/src/routes/(app)/storage/+page.svelte @@ -7,7 +7,6 @@ import { navigating } from '$app/state'; import * as m from '$lib/paraglide/messages.js'; import { getStorageState } from '$lib/storage/context.js'; - import { storageRestoreTabsEnabled } from '$lib/client/feature-flags.js'; import { type PersistedTabsState } from '$lib/storage/tabs.svelte.js'; import { LS_TABS } from '$lib/storage/persistence.js'; import IconTabOutline from 'virtual:icons/material-symbols/tab-outline'; @@ -31,7 +30,7 @@ let savedTabs = $state(null); $effect(() => { - if (!browser || !storageRestoreTabsEnabled) return; + if (!browser) return; try { const raw = localStorage.getItem(LS_TABS); @@ -58,7 +57,7 @@ let restoreClicked = $state(false); $effect(() => { - if (!browser || !storageRestoreTabsEnabled) return; + if (!browser) return; const to = navigating?.to?.url.pathname; if (!to || !to.startsWith('/storage/') || to === '/storage') return; // Navigation to a bucket sub-path is starting — if the user did not diff --git a/src/test/setup-client.ts b/src/test/setup-client.ts index bd25aa29..c0d73bfd 100644 --- a/src/test/setup-client.ts +++ b/src/test/setup-client.ts @@ -1,21 +1,17 @@ import { vi, beforeEach } from 'vitest'; -// Mock client feature flags so tests don't try to resolve the SvelteKit -// $env/dynamic/public virtual module, which is undefined in the Vitest -// browser environment. Values match the defaults used in production when no -// env vars are set. +// Mock client feature flags and config so tests don't try to resolve the +// SvelteKit $env/dynamic/public virtual module, which is undefined in the +// Vitest browser environment. Values match the production defaults. vi.mock('$lib/client/feature-flags.js', () => ({ - storageRestoreTabsEnabled: false, - infiniteScrollEnabled: true, + storageTransferEnabled: false +})); +vi.mock('$lib/client/config.js', () => ({ allowedPageSizes: [25, 50, 100], defaultPageSize: 25, maxRecentFiles: 15, maxEditableFileSize: 5 * 1024 * 1024, - uploadConcurrency: 3, - storageCutCopyEnabled: false, - storagePasteEnabled: false, - storageRenameEnabled: false, - storageMoveEnabled: false + uploadConcurrency: 3 })); // Prevent components from auto-submitting forms in browser tests.