From aa6f3889d42f3cc0759984c31e53b1b02c6433a7 Mon Sep 17 00:00:00 2001 From: Lin Zhang Date: Sat, 14 Nov 2026 19:07:41 +0900 Subject: [PATCH 1/3] feat(torrent): move a torrent's files to another folder --- AGENTS.md | 21 ++++ docs/developers.md | 1 + docs/torrent.md | 17 +++ .../kotlin/com/linroid/ketch/core/Ketch.kt | 60 ++++++++++ .../ketch/core/engine/SourceExtension.kt | 18 +++ .../torrent/TorrentRelocateRequest.kt | 11 ++ .../ketch/endpoints/torrent/Torrents.kt | 6 + .../ketch/remote/RemoteTorrentController.kt | 11 ++ .../linroid/ketch/server/DestinationGuard.kt | 13 ++ .../com/linroid/ketch/server/KetchServer.kt | 2 +- .../linroid/ketch/server/api/TorrentRoutes.kt | 17 ++- .../linroid/ketch/server/TorrentRoutesTest.kt | 70 ++++++++++- .../ketch/torrent/api/TorrentController.kt | 17 +++ .../ketch/torrent/TorrentDownloadSource.kt | 81 ++++++++++++- .../ketch/torrent/TorrentSourceState.kt | 5 + .../torrent/control/KetchTorrentController.kt | 45 +++++++ .../torrent/control/TorrentControlSource.kt | 13 ++ .../torrent/engine/KotlinTorrentEngine.kt | 3 +- .../ketch/torrent/engine/TorrentEngine.kt | 2 + .../ketch/torrent/storage/TorrentMove.kt | 38 ++++++ .../torrent/storage/TorrentPieceStore.kt | 13 ++ .../torrent/TorrentDownloadSourceTest.kt | 2 +- .../torrent/TorrentSeedingControlTest.kt | 112 ++++++++++++++++++ .../ketch/torrent/storage/TorrentMoveTest.kt | 79 ++++++++++++ 24 files changed, 647 insertions(+), 10 deletions(-) create mode 100644 library/endpoints/src/commonMain/kotlin/com/linroid/ketch/endpoints/torrent/TorrentRelocateRequest.kt create mode 100644 library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentMove.kt create mode 100644 library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/storage/TorrentMoveTest.kt diff --git a/AGENTS.md b/AGENTS.md index fdf91506f..a7d080bf3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1362,6 +1362,27 @@ docker/ # Docker image of `ketch server` (Dockerfile, entrypoint, compose metainfo's usable trackers, else a magnet's. Trackers Ketch adds to public torrents are never listed. The inspector's Trackers tab (`TrackersTab`, `trackerView`), offered for torrents on devices with the capability, asks again every 3 s while it shows +- Moving files: `TorrentController.relocate(taskId, directory)` (REST + `PUT /api/torrents/{id}/location` with a `TorrentRelocateRequest`, answering the snapshot; + capability `relocate`) moves what a task saved into another folder under the name it has. The + controller pauses a downloading or queued task and stops a seeder, then `SourceHost.relocate` + (core, under the task's control lock, with no execution active) reserves `/`, + refusing a relative folder, the folder the task is in, one inside its files, a name that is + taken or another unfinished task's output, runs the source's move and saves the new + `outputPath` and source state in one record write, also in a `Completed` state. + `TorrentDownloadSource.relocate` moves the tree (`moveTorrentTree`: a rename, else between + volumes a copy that is deleted again when it fails, then the original deleted), deletes the + bookkeeping beside the old place (the v1 `.ketch--` folder, the v2 creation log) + and answers the state with the new `savePath`, no checkpoint and `relocated`, so the next run + checks the files where they are and takes them as the task's own: v2 and hybrid stores through + `adoptExisting`, and v1 stores through `TorrentPieceStore(adoptExisting)`, which records the + selected files in place and their folders as owned as it initializes (also for a task added + to use files already there). The task then goes on: resumed, or reopened to be checked and + seed again (`prepareRecheck`, `SourceHost.reopen`); a paused one is checked when it next + starts. Payload counters start again with the new checkpoint, and pieces kept for skipped + files in the v1 sidecar are downloaded again when their files are chosen. The server lets a + caller move only where it may save (`DestinationGuard.checkMove`, 403 `path_rejected`). The + apps have no control for it yet - Editing trackers: `TorrentController.editTrackers(taskId, TorrentTrackerEdit(add, remove))` (REST `PUT /api/torrents/{id}/trackers`, answering the list; capability `tracker-edit`) removes the trackers at the endpoints `remove` names and appends each announce URL of `add` as a tier diff --git a/docs/developers.md b/docs/developers.md index cef4ecd50..0f3a98bb5 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -181,6 +181,7 @@ These exist on servers listing `torrent.control`: | `GET /api/torrents/{id}/priority`, `PUT` | The files it downloads before its others, as `{"fileIds": [...]}`; `PUT` sets them, an empty list for the usual order (`file-priority` capability) | | `POST /api/torrents/{id}/recheck` | Check its files against the torrent's hashes again and download what is missing; answers its snapshot (`recheck` capability) | | `GET /api/torrents/{id}/trackers` | Its own trackers in tier order: endpoint (never the path), state (`idle`, `announcing`, `working`, `failed`, `timed-out`), peers named, seeders, leechers and what the tracker said (`trackers` capability) | +| `PUT /api/torrents/{id}/location` | Moves its files into `{"directory": absolute folder}` under the name they have and answers the snapshot; `path_rejected` for a folder callers may not save to, `invalid_input` for a folder the files cannot move into, `storage_failure` when the move fails (`relocate` capability) | | `PUT /api/torrents/{id}/trackers` | Adds and removes its own trackers: `{"add": [announce URLs], "remove": [endpoints]}`, answering the new list; `invalid_input` for a URL that is not `http`, `https` or `udp` (`tracker-edit` capability) | | `GET /api/torrents/{id}/export?metainfo=true` | Its info hashes and magnet link (none for a private torrent), with its `.torrent` file as Base64 when `metainfo` is true (`export` capability) | | `GET /api/torrents/{id}/events` | Deprecated: SSE `snapshot` events of it; the event stream carries them | diff --git a/docs/torrent.md b/docs/torrent.md index dbc359e94..ea5e46733 100644 --- a/docs/torrent.md +++ b/docs/torrent.md @@ -547,6 +547,23 @@ trackers, and a magnet link's trackers can be edited only once its metadata is f `{"add": ["udp://tracker.example:6969/announce"], "remove": ["udp://old.example:1337"]}`, which answers the new list; SDK: `TorrentController.editTrackers(taskId, TorrentTrackerEdit(...))`. +### Moving a torrent's files + +A torrent's files can be moved to another folder without adding it again: +`PUT /api/torrents/{id}/location` with `{"directory": "/absolute/folder"}`, on devices listing +the `relocate` capability (SDK: `TorrentController.relocate(taskId, directory)`); the apps have +no button for it yet. The files keep their name inside the new folder. A downloading torrent +stops for the move and goes on after it; a finished one is checked at its new place and seeds +again if it was seeding; a paused one is checked when it next starts. On the same volume the move +is a rename; between volumes the files are copied and the originals deleted once the copy is +complete, which takes as long as copying them does, and a copy that fails is removed again, +leaving the files where they were. The move is refused for a folder that is not an absolute +path, the folder the torrent is in already, a folder inside its own files, and a folder that +already has something of the torrent's name. A server lets callers move files only into folders +they may save to (see [the daemon's folders](developers.md)). After a move the torrent's upload +and download totals start again, and the parts of skipped files it had kept are downloaded +again if you choose those files later. + ### How download settings apply to torrents Torrent tasks follow the same download settings as HTTP and FTP tasks, with these differences: diff --git a/library/core/src/commonMain/kotlin/com/linroid/ketch/core/Ketch.kt b/library/core/src/commonMain/kotlin/com/linroid/ketch/core/Ketch.kt index c866a8f72..4ccccb114 100644 --- a/library/core/src/commonMain/kotlin/com/linroid/ketch/core/Ketch.kt +++ b/library/core/src/commonMain/kotlin/com/linroid/ketch/core/Ketch.kt @@ -58,6 +58,7 @@ import com.linroid.ketch.core.engine.decodeState import com.linroid.ketch.core.engine.unknownFilesMessage import com.linroid.ketch.core.file.DefaultFileNameResolver import com.linroid.ketch.core.file.FileNameResolver +import com.linroid.ketch.core.file.OutputPathReservations import com.linroid.ketch.core.file.requireDownloadDirectory import com.linroid.ketch.core.geo.CountryLookup import com.linroid.ketch.core.speed.SpeedModeKeeper @@ -106,6 +107,7 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull +import okio.Path.Companion.toPath import kotlin.concurrent.Volatile import kotlin.concurrent.atomics.AtomicBoolean import kotlin.concurrent.atomics.ExperimentalAtomicApi @@ -679,6 +681,12 @@ class Ketch( } } + override suspend fun relocate( + task: DownloadTask, + directory: String, + move: suspend (from: String, to: String, state: SourceState?) -> SourceState?, + ): String = relocateTask(requireHandle(task), directory, move) + override suspend fun reopen(task: DownloadTask): Boolean { val handle = handleOf(task) ?: return false return reopenCompleted(handle) @@ -712,6 +720,58 @@ class Ketch( } } + /** What [SourceHost.relocate] does for [handle]. */ + private suspend fun relocateTask( + handle: RealDownloadTask, + directory: String, + move: suspend (from: String, to: String, state: SourceState?) -> SourceState?, + ): String { + val taskId = handle.taskId + coordinator.awaitCompletion(taskId) + return handle.controlLock.withLock { + val record = handle.record.value + val from = record.outputPath + check(from != null && record.state != TaskState.CANCELED && !coordinator.isActive(taskId)) { + "The download has nothing to move, or is running" + } + check("://" !in from && "://" !in directory) { "Document folders cannot be moved between" } + val source = from.toPath().normalized() + val folder = directory.toPath().normalized() + require(folder.isAbsolute) { "The folder to move to must be an absolute path" } + val target = folder / source.name + require(target != source) { "The download is in that folder already" } + require(generateSequence(folder) { it.parent }.none { it == source }) { + "A download cannot be moved into itself" + } + val to = target.toString() + val taken = unfinishedOutputs(taskId).any { other -> + "://" !in other && other.toPath().normalized().let { owned -> + owned == target || generateSequence(owned.parent) { it.parent }.any { it == target } || + generateSequence(target.parent) { it.parent }.any { it == owned } + } + } + require(!taken && OutputPathReservations.reserveUnique(to).also { reserved -> + if (reserved != to) OutputPathReservations.release(reserved) + } == to) { "The folder already has something of that name" } + try { + // Saved even when the caller is cancelled once the files are at their new place. + withContext(NonCancellable) { + val state = move(from, to, record.sourceState) + handle.record.update { + it.copy(outputPath = to, sourceState = state, updatedAt = Clock.System.now()) + } + (handle.mutableState.value as? DownloadState.Completed)?.let { completed -> + handle.mutableState.value = completed.copy(outputPath = to) + } + } + } finally { + OutputPathReservations.release(to) + } + log.i { "Moved the files of taskId=$taskId" } + to + } + } + /** [task] when it is a task of this instance. */ private fun handleOf(task: DownloadTask): RealDownloadTask? = (task as? RealDownloadTask)?.takeIf { real -> _tasks.value.any { it === real } } diff --git a/library/core/src/commonMain/kotlin/com/linroid/ketch/core/engine/SourceExtension.kt b/library/core/src/commonMain/kotlin/com/linroid/ketch/core/engine/SourceExtension.kt index e90f624a7..bdc2d3762 100644 --- a/library/core/src/commonMain/kotlin/com/linroid/ketch/core/engine/SourceExtension.kt +++ b/library/core/src/commonMain/kotlin/com/linroid/ketch/core/engine/SourceExtension.kt @@ -141,4 +141,22 @@ interface SourceHost { * task is the caller's to stop first. */ suspend fun reopen(task: DownloadTask): Boolean = false + + /** + * Moves what [task] saved into the folder [directory], under the name it has: [move] gets the + * path it saves at, the path beside [directory] reserved for it and its saved source state, + * moves the files and answers the source state to save with the new path. Runs under the + * task's control lock, so nothing selects, shares or reopens it meanwhile. Returns the new + * path. + * + * @throws IllegalStateException for a task that runs, was canceled or saved nowhere yet, or + * that saves to a document URI + * @throws IllegalArgumentException when [directory] is where the task is already, lies inside + * what it saved, or the name is taken there, by a file or by another download + */ + suspend fun relocate( + task: DownloadTask, + directory: String, + move: suspend (from: String, to: String, state: SourceState?) -> SourceState?, + ): String = throw IllegalStateException("This download cannot be moved") } diff --git a/library/endpoints/src/commonMain/kotlin/com/linroid/ketch/endpoints/torrent/TorrentRelocateRequest.kt b/library/endpoints/src/commonMain/kotlin/com/linroid/ketch/endpoints/torrent/TorrentRelocateRequest.kt new file mode 100644 index 000000000..1b4acf08a --- /dev/null +++ b/library/endpoints/src/commonMain/kotlin/com/linroid/ketch/endpoints/torrent/TorrentRelocateRequest.kt @@ -0,0 +1,11 @@ +package com.linroid.ketch.endpoints.torrent + +import kotlinx.serialization.Serializable + +/** + * Body of `PUT /api/torrents/{id}/location`. + * + * @property directory the folder, an absolute path on the server, the task's files move into + */ +@Serializable +data class TorrentRelocateRequest(val directory: String) diff --git a/library/endpoints/src/commonMain/kotlin/com/linroid/ketch/endpoints/torrent/Torrents.kt b/library/endpoints/src/commonMain/kotlin/com/linroid/ketch/endpoints/torrent/Torrents.kt index 04b20b128..192a43a8d 100644 --- a/library/endpoints/src/commonMain/kotlin/com/linroid/ketch/endpoints/torrent/Torrents.kt +++ b/library/endpoints/src/commonMain/kotlin/com/linroid/ketch/endpoints/torrent/Torrents.kt @@ -21,6 +21,7 @@ import kotlinx.serialization.Serializable * its `.torrent` file) * - `GET /api/torrents/{id}/priority`, `PUT` — the files it downloads before its others * - `POST /api/torrents/{id}/recheck` — check its files again and download what is missing + * - `PUT /api/torrents/{id}/location` — moves its files into another folder * - `GET /api/torrents/{id}/trackers` — its own trackers and how each answers * - `PUT /api/torrents/{id}/trackers` — adds and removes trackers (`TorrentTrackerEdit`) * - `GET /api/torrents/{id}/events` — SSE stream of its snapshots @@ -96,6 +97,11 @@ data class Torrents(val parent: Api = Api()) { @Resource("recheck") data class Recheck(val parent: ById) + /** Moves the task's files into another folder: `PUT` with a `TorrentRelocateRequest`. */ + @Serializable + @Resource("location") + data class Location(val parent: ById) + /** * The task's own trackers in tier order, a list of `TorrentTracker`; `PUT` with a * `TorrentTrackerEdit` changes them and answers the list. diff --git a/library/remote/src/commonMain/kotlin/com/linroid/ketch/remote/RemoteTorrentController.kt b/library/remote/src/commonMain/kotlin/com/linroid/ketch/remote/RemoteTorrentController.kt index 3010de474..f9a4213ea 100644 --- a/library/remote/src/commonMain/kotlin/com/linroid/ketch/remote/RemoteTorrentController.kt +++ b/library/remote/src/commonMain/kotlin/com/linroid/ketch/remote/RemoteTorrentController.kt @@ -4,6 +4,7 @@ import com.linroid.ketch.api.Interest import com.linroid.ketch.api.log.KetchLogger import com.linroid.ketch.endpoints.model.ErrorResponse import com.linroid.ketch.endpoints.torrent.TorrentPriorityRequest +import com.linroid.ketch.endpoints.torrent.TorrentRelocateRequest import com.linroid.ketch.endpoints.torrent.TorrentSeedingRequest import com.linroid.ketch.endpoints.torrent.TorrentSequentialRequest import com.linroid.ketch.endpoints.torrent.TorrentSelectionRequest @@ -371,6 +372,16 @@ internal class RemoteTorrentController( } } + override suspend fun relocate(taskId: String, directory: String) { + requireCapability(TorrentCapability.RELOCATE) + call(notFoundIsNull = false) { + httpClient.put(Torrents.ById.Location(parent = Torrents.ById(id = taskId))) { + contentType(ContentType.Application.Json) + setBody(TorrentRelocateRequest(directory)) + } + } + } + override suspend fun trackers(taskId: String): List? { requireCapability(TorrentCapability.TRACKERS) return call(notFoundIsNull = true) { diff --git a/library/server/src/main/kotlin/com/linroid/ketch/server/DestinationGuard.kt b/library/server/src/main/kotlin/com/linroid/ketch/server/DestinationGuard.kt index 3c6b93c7d..9031fd1dd 100644 --- a/library/server/src/main/kotlin/com/linroid/ketch/server/DestinationGuard.kt +++ b/library/server/src/main/kotlin/com/linroid/ketch/server/DestinationGuard.kt @@ -163,6 +163,19 @@ internal class DestinationGuard( } } + /** + * Checks that a download's files may be moved into [directory], as a download may be saved + * there. + * + * @throws PathRejectedException if the folder lies outside the folders + */ + suspend fun checkMove(directory: String) { + val status = ketch.status() + if (passesAsUri(directory) || !policy(status.system.downloadDirectory).contains(directory)) { + throw PathRejectedException(directory) + } + } + /** * Checks that removing [task] with its files deletes nothing outside the folders. * diff --git a/library/server/src/main/kotlin/com/linroid/ketch/server/KetchServer.kt b/library/server/src/main/kotlin/com/linroid/ketch/server/KetchServer.kt index c971e5c8a..67f3ddf9a 100644 --- a/library/server/src/main/kotlin/com/linroid/ketch/server/KetchServer.kt +++ b/library/server/src/main/kotlin/com/linroid/ketch/server/KetchServer.kt @@ -606,7 +606,7 @@ private fun Route.apiRoutes( ) { serverRoutes(ketch, destinations, onConfigChanged) downloadRoutes(ketch, destinations) - ketch.controller(TorrentController)?.let { torrentRoutes(ketch, it) } + ketch.controller(TorrentController)?.let { torrentRoutes(ketch, it, destinations) } ketch.controller(CountryDatabase)?.let { countryRoutes(it) } ketch.controller(SpeedModes)?.let { speedRoutes(it) } connectionRoutes(ketch, connectionStreams) diff --git a/library/server/src/main/kotlin/com/linroid/ketch/server/api/TorrentRoutes.kt b/library/server/src/main/kotlin/com/linroid/ketch/server/api/TorrentRoutes.kt index b4157415b..66ae8e8eb 100644 --- a/library/server/src/main/kotlin/com/linroid/ketch/server/api/TorrentRoutes.kt +++ b/library/server/src/main/kotlin/com/linroid/ketch/server/api/TorrentRoutes.kt @@ -4,6 +4,7 @@ import com.linroid.ketch.api.KetchApi import com.linroid.ketch.api.log.KetchLogger import com.linroid.ketch.endpoints.model.ErrorResponse import com.linroid.ketch.endpoints.torrent.TorrentPriorityRequest +import com.linroid.ketch.endpoints.torrent.TorrentRelocateRequest import com.linroid.ketch.endpoints.torrent.TorrentSeedingRequest import com.linroid.ketch.endpoints.torrent.TorrentSequentialRequest import com.linroid.ketch.endpoints.torrent.TorrentSelectionRequest @@ -16,6 +17,7 @@ import com.linroid.ketch.torrent.api.TorrentInterest import com.linroid.ketch.torrent.api.TorrentPageRequest import com.linroid.ketch.torrent.api.TorrentSettingsChange import com.linroid.ketch.torrent.api.TorrentTrackerEdit +import com.linroid.ketch.server.DestinationGuard import io.ktor.http.HttpStatusCode import io.ktor.server.resources.get import io.ktor.server.resources.post @@ -45,7 +47,11 @@ private const val MAX_CURSOR_LENGTH = 4096 * The deprecated per-torrent event stream follows [TorrentInterest] on [ketch]; clients of the * event stream follow it there. */ -internal fun Route.torrentRoutes(ketch: KetchApi, controller: TorrentController) { +internal fun Route.torrentRoutes( + ketch: KetchApi, + controller: TorrentController, + destinations: DestinationGuard, +) { get { call.respond(controller.capabilities()) } @@ -152,6 +158,15 @@ internal fun Route.torrentRoutes(ketch: KetchApi, controller: TorrentController) call.respond(controller.snapshot(taskId) ?: throw notFound(taskId)) } + put { resource -> + val taskId = resource.parent.id + val body = call.receiveJson() + log.d { "PUT /api/torrents/$taskId/location" } + destinations.checkMove(body.directory) + controller.relocate(taskId, body.directory) + call.respond(controller.snapshot(taskId) ?: throw notFound(taskId)) + } + get { resource -> val taskId = resource.parent.id log.d { "GET /api/torrents/$taskId/trackers" } diff --git a/library/server/src/test/kotlin/com/linroid/ketch/server/TorrentRoutesTest.kt b/library/server/src/test/kotlin/com/linroid/ketch/server/TorrentRoutesTest.kt index 9c4f4e219..92e3267c0 100644 --- a/library/server/src/test/kotlin/com/linroid/ketch/server/TorrentRoutesTest.kt +++ b/library/server/src/test/kotlin/com/linroid/ketch/server/TorrentRoutesTest.kt @@ -3,11 +3,13 @@ package com.linroid.ketch.server import com.linroid.ketch.api.ControlKey import com.linroid.ketch.api.Interest import com.linroid.ketch.api.KetchApi +import com.linroid.ketch.api.KetchStatus import com.linroid.ketch.api.SpeedLimit import com.linroid.ketch.endpoints.model.ErrorResponse +import com.linroid.ketch.endpoints.torrent.TorrentRelocateRequest import com.linroid.ketch.endpoints.torrent.TorrentSeedingRequest -import com.linroid.ketch.endpoints.torrent.TorrentSequentialRequest import com.linroid.ketch.endpoints.torrent.TorrentSelectionRequest +import com.linroid.ketch.endpoints.torrent.TorrentSequentialRequest import com.linroid.ketch.torrent.api.TorrentActivity import com.linroid.ketch.torrent.api.TorrentCapabilities import com.linroid.ketch.torrent.api.TorrentCommandContext @@ -28,6 +30,7 @@ import com.linroid.ketch.torrent.api.TorrentRevision import com.linroid.ketch.torrent.api.TorrentSettingsChange import com.linroid.ketch.torrent.api.TorrentSnapshot import com.linroid.ketch.torrent.api.TorrentTracker +import com.linroid.ketch.torrent.api.TorrentTrackerEdit import com.linroid.ketch.torrent.api.TorrentTrackerState import com.linroid.ketch.torrent.api.TorrentUploadSetting import io.ktor.client.request.get @@ -165,6 +168,25 @@ class TorrentRoutesTest { first = fileIds } + val moves = mutableListOf>() + + override suspend fun relocate(taskId: String, directory: String) { + moves += taskId to directory + } + + val edits = mutableListOf() + + override suspend fun editTrackers( + taskId: String, + edit: TorrentTrackerEdit, + ): List { + if (edit.add.any { it.startsWith("wss:") }) { + throw TorrentCommandException(TorrentCommandError.INVALID_INPUT, "Invalid trackers") + } + edits += edit + return edit.add.mapIndexed { index, _ -> TorrentTracker("udp://added.example:1", index) } + } + val rechecked = mutableListOf() override suspend fun recheck(taskId: String) { @@ -433,6 +455,52 @@ class TorrentRoutesTest { assertEquals("""{"fileIds":["2","5"]}""", client.get("/api/torrents/t1/priority").bodyAsText()) } + @Test + fun location_movesInsideTheDownloadFolder_andRefusesOtherFolders() = testApplication { + val controller = ScriptedController() + application { with(controlledServer(controller)) { configureServer() } } + val status = json.decodeFromString(client.get("/api/status").bodyAsText()) + val inside = status.system.downloadDirectory.trimEnd('/', '\\') + "/moved" + + val moved = client.put("/api/torrents/t1/location") { + contentType(ContentType.Application.Json) + setBody(json.encodeToString(TorrentRelocateRequest(inside))) + } + // Without a token, callers stay inside the download folder. + val outside = client.put("/api/torrents/t1/location") { + contentType(ContentType.Application.Json) + setBody(json.encodeToString(TorrentRelocateRequest("/somewhere/else/entirely"))) + } + + assertEquals(HttpStatusCode.OK, moved.status) + assertEquals("t1", json.decodeFromString(moved.bodyAsText()).taskId) + assertEquals(HttpStatusCode.Forbidden, outside.status) + assertEquals(listOf("t1" to inside), controller.moves) + } + + @Test + fun trackerEdit_reachesTheController_andNeverEchoesARefusedUrl() = testApplication { + val controller = ScriptedController() + application { with(controlledServer(controller)) { configureServer() } } + + val edited = client.put("/api/torrents/t1/trackers") { + contentType(ContentType.Application.Json) + setBody("""{"add":["udp://t.example:6969/announce"],"remove":["udp://old.example:1"]}""") + } + val refused = client.put("/api/torrents/t1/trackers") { + contentType(ContentType.Application.Json) + setBody("""{"add":["wss://t.example/announce?passkey=secret"]}""") + } + + assertEquals(HttpStatusCode.OK, edited.status) + assertEquals(listOf(TorrentTracker("udp://added.example:1", 0)), + json.decodeFromString>(edited.bodyAsText())) + assertEquals(listOf(TorrentTrackerEdit(listOf("udp://t.example:6969/announce"), + listOf("udp://old.example:1"))), controller.edits) + assertEquals(HttpStatusCode.BadRequest, refused.status) + assertFalse("secret" in refused.bodyAsText()) + } + @Test fun recheck_reachesTheController_andAnswersTheSnapshot() = testApplication { val controller = ScriptedController() diff --git a/library/torrent-api/src/commonMain/kotlin/com/linroid/ketch/torrent/api/TorrentController.kt b/library/torrent-api/src/commonMain/kotlin/com/linroid/ketch/torrent/api/TorrentController.kt index 75ebde2b9..da3f0dbf6 100644 --- a/library/torrent-api/src/commonMain/kotlin/com/linroid/ketch/torrent/api/TorrentController.kt +++ b/library/torrent-api/src/commonMain/kotlin/com/linroid/ketch/torrent/api/TorrentController.kt @@ -144,6 +144,23 @@ interface TorrentController { message = "Rechecking is unavailable", ) + /** + * Moves the task's files into the folder [directory], an absolute path on the device that + * runs it, where they keep their name, and checks them there: a downloading task stops for + * the move and goes on after it, a completed one is checked and seeds again if it was meant + * to, and others are checked when they next start. Between volumes the files are copied, which + * takes as long as copying them does. Fails with [TorrentCommandError.NOT_FOUND] for unknown + * and non-torrent tasks, [TorrentCommandError.INVALID_STATE] for a task that was canceled or + * has saved nothing yet, [TorrentCommandError.INVALID_INPUT] for a folder that is not an + * absolute path, is where the files are, lies inside them or holds something of their name, + * and [TorrentCommandError.STORAGE_FAILURE] when the files cannot be moved, which leaves them + * where they were. Advertised as [TorrentCapability.RELOCATE]. + */ + suspend fun relocate(taskId: String, directory: String): Unit = throw TorrentCommandException( + error = TorrentCommandError.UNSUPPORTED, + message = "Moving files is unavailable", + ) + /** * The task's own trackers in tier order and how each answers, or `null` for unknown and * non-torrent tasks: live while the torrent runs or seeds, and [TorrentTrackerState.IDLE] diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentDownloadSource.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentDownloadSource.kt index 85b0382fe..afc329e82 100644 --- a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentDownloadSource.kt +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentDownloadSource.kt @@ -65,6 +65,7 @@ import com.linroid.ketch.torrent.metainfo.V2InfoHash import com.linroid.ketch.torrent.net.ResponseTooLargeException import com.linroid.ketch.torrent.net.TorrentHttp import com.linroid.ketch.torrent.storage.TorrentCheckpoint +import com.linroid.ketch.torrent.storage.moveTorrentTree import com.linroid.ketch.torrent.storage.TorrentPieceStore import com.linroid.ketch.torrent.storage.TorrentStorageException import com.linroid.ketch.torrent.storage.TorrentV2Checkpoint @@ -108,7 +109,9 @@ import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull import kotlinx.serialization.KSerializer +import okio.FileSystem import okio.IOException +import okio.Path import okio.Path.Companion.toPath import kotlin.concurrent.atomics.AtomicBoolean import kotlin.concurrent.atomics.AtomicLong @@ -301,6 +304,13 @@ class TorrentDownloadSource( override fun withTrackers(state: SourceState, tiers: List>) = source.withTrackers(state, tiers) + + override suspend fun relocate( + taskId: String, + from: String, + to: String, + sourceState: SourceState?, + ) = source.relocate(taskId, from, to, sourceState) override val restoresSeeding: Boolean get() = source.restoresSeeding override val sharingResumes: StateFlow get() = source.sharingResumes @@ -380,6 +390,7 @@ class TorrentDownloadSource( add(TorrentCapability.FILE_PRIORITY.wireName) add(TorrentCapability.IP_FILTER.wireName) add(TorrentCapability.TRACKER_EDIT.wireName) + add(TorrentCapability.RELOCATE.wireName) if (lsdSupported) add(TorrentCapability.LOCAL_DISCOVERY.wireName) // Sharing follows the live upload setting, paused or not. if (uploadPolicy.load() == TorrentUploadPolicy.SEED_AFTER_COMPLETION) { @@ -453,6 +464,64 @@ class TorrentDownloadSource( internal fun withSequential(state: SourceState, sequential: Boolean): SourceState = changed(state) { it.copy(sequential = sequential) } + /** + * Whether a run of [context] that continues from [previous] takes the files at its paths as + * the task's own: a check of its files takes them as they are now, replaced or gone, Ketch + * gave the run a path that may hold files only because the request asked for it, and a task + * whose files were moved finds them where they were moved to. + */ + private fun adopts(context: DownloadContext, previous: TorrentSourceState?): Boolean = + context.taskId in rechecks.value || previous?.relocated == true || + context.request.fileConflict == FileConflict.OVERWRITE + + /** + * What [TorrentControlSource.relocate] does: moves what the stopped task [taskId] saved at + * [from] to [to] and deletes the bookkeeping it kept beside [from], which describes the files + * where they were. Returns [sourceState] saving at [to] without its checkpoint, so the next + * run checks the files where they are now and takes them as the task's own. + * + * @throws KetchError.Disk when the files cannot be moved; they then stay at [from] + */ + internal suspend fun relocate( + taskId: String, + from: String, + to: String, + sourceState: SourceState?, + ): SourceState? { + check(tasks.session(taskId) == null && !tasks.isReserved(taskId)) { + "Stop the torrent before moving its files" + } + val state = sourceState?.let { decodeState(it) } + withContext(TorrentDispatchers.disk) { + val files = torrentSystemFileSystem + val source = from.toPath() + val target = to.toPath() + try { + target.parent?.let { files.createDirectories(it) } + // A task that created nothing yet has nothing to move. + if (files.metadataOrNull(source) != null) moveTorrentTree(files, source, target) + } catch (e: IOException) { + throw KetchError.Disk(e) + } + val beside = source.parent ?: return@withContext + val kept = listOfNotNull( + state?.takeIf { it.version != 3 }?.let { beside / ".ketch-${it.infoHash}-$taskId" }, + v2CreationLog(source, taskId), + ) + for (path in kept) { + try { + files.deleteRecursively(path, mustExist = false) + } catch (e: IOException) { + log.w { "Kept old bookkeeping of taskId=$taskId: ${e.describeCauses()}" } + } + } + } + stateMutex.withLock { states.remove(taskId) } + seedStates.update { it - taskId } + log.i { "Moved the files of torrent taskId=$taskId" } + return state?.let { encodeState(it.copy(savePath = to, resumeData = "", relocated = true)) } + } + /** What [TorrentControlSource.withTrackers] answers. */ internal fun withTrackers(state: SourceState, tiers: List>): SourceState = changed(state) { it.copy(trackerTiers = tiers) } @@ -1816,7 +1885,8 @@ class TorrentDownloadSource( val state = TorrentSourceState(hash, total, previous?.resumeData ?: "", selected, output, previous?.metainfo.orEmpty(), version = 2, privacy = privacy, sequential = previous?.sequential == true, - priorityFileIds = previous?.priorityFileIds.orEmpty(), trackerTiers = trackers) + priorityFileIds = previous?.priorityFileIds.orEmpty(), trackerTiers = trackers, + relocated = previous?.relocated == true) log.i { "Starting torrent taskId=${context.taskId} (${logHash(hash)}): " + "files=${selected.size}/${table.files.size}, totalBytes=$total, output=$output, " + @@ -1827,7 +1897,8 @@ class TorrentDownloadSource( selected.mapTo(LinkedHashSet()) { it.toInt() }, context.url.takeIf { it.startsWith("magnet:", true) }, previous?.resumeData?.takeIf { it.isNotEmpty() }?.let(::decodeBase64), context.throttle, - privacy = privacy, connections = context.connections)) + privacy = privacy, connections = context.connections, + adoptExisting = adopts(context, previous))) } } @@ -1854,7 +1925,8 @@ class TorrentDownloadSource( val state = TorrentSourceState(hash, total, previous?.resumeData.orEmpty(), selected, output, previous?.metainfo.orEmpty(), version = 3, privacy = privacy, sequential = previous?.sequential == true, - priorityFileIds = previous?.priorityFileIds.orEmpty(), trackerTiers = trackers) + priorityFileIds = previous?.priorityFileIds.orEmpty(), trackerTiers = trackers, + relocated = previous?.relocated == true) log.i { "Starting v2 torrent taskId=${context.taskId} (${logHash(hash)}): " + "files=${selected.size}/${table.files.size}, totalBytes=$total, output=$output, " + @@ -1873,8 +1945,7 @@ class TorrentDownloadSource( connections = context.connections, // A check of the task's files takes them as they are now, replaced or gone, and Ketch // gave the run a path that may hold files only because the request asked for it. - adoptExisting = context.taskId in rechecks.value || - context.request.fileConflict == FileConflict.OVERWRITE, + adoptExisting = adopts(context, previous), )) } } diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentSourceState.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentSourceState.kt index 2ff8fe095..6f422d8d7 100644 --- a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentSourceState.kt +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentSourceState.kt @@ -38,6 +38,11 @@ data class TorrentSourceState internal constructor( * see [TorrentDownloadSource.editTrackers]. `null` for the torrent's. */ internal val trackerTiers: List>? = null, + /** + * Whether the task's files were moved to where it saves now; see + * [TorrentDownloadSource.relocate]. Its sessions take the files they find there as its own. + */ + internal val relocated: Boolean = false, ) { init { require(version in 1..3) { "Unsupported torrent resume version" } diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/control/KetchTorrentController.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/control/KetchTorrentController.kt index 0fe1e299e..5737bb042 100644 --- a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/control/KetchTorrentController.kt +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/control/KetchTorrentController.kt @@ -44,6 +44,7 @@ import com.linroid.ketch.torrent.engine.TorrentPayloadCounters import com.linroid.ketch.torrent.net.TorrentIpFilter import kotlin.time.Duration.Companion.seconds import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.cancelChildren import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope @@ -60,6 +61,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext import kotlinx.serialization.Serializable import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json @@ -257,6 +259,49 @@ internal class KetchTorrentController( controlChanged() } + override suspend fun relocate(taskId: String, directory: String) { + val task = torrentTask(taskId) ?: throw notFound() + requireCapability(TorrentCapability.RELOCATE, TorrentCommandError.UNSUPPORTED) + val state = task.state.value + if (state is DownloadState.Canceled) { + throw TorrentCommandException( + TorrentCommandError.INVALID_STATE, + "A canceled download has no files to move", + ) + } + val running = state is DownloadState.Downloading || state is DownloadState.Queued + val completed = state is DownloadState.Completed + // Read before the seeder stops: a task its user stopped seeding stays stopped. + val meantToSeed = completed && seedingOf(task, recordOf(task).control ?: TaskControl()) + var moved = false + try { + mapErrors { + if (running) task.pause() + if (completed) source.stopSeeding(taskId) + host.relocate(task, directory) { from, to, saved -> + source.relocate(taskId, from, to, saved) + } + moved = true + } + } finally { + withContext(NonCancellable) { + // Checked where they are now; a move that failed left them as they were. + if (moved) source.prepareRecheck(taskId, seedAfter = !completed || meantToSeed) + when { + running -> runCatching { task.resume() }.onFailure { e -> + log.w { "taskId=$taskId did not start again after its move: ${e.describeCauses()}" } + } + completed && moved -> if (!host.reopen(task)) { + log.w { "taskId=$taskId changed while it moved; it is checked when it next starts" } + } + completed && meantToSeed -> seed(task) + } + } + } + log.i { "Moved taskId=$taskId" } + controlChanged() + } + override suspend fun trackers(taskId: String): List? { val task = torrentTask(taskId) ?: return null requireCapability(TorrentCapability.TRACKERS, TorrentCommandError.UNSUPPORTED) diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/control/TorrentControlSource.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/control/TorrentControlSource.kt index 8da0cc27a..c6d222e6d 100644 --- a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/control/TorrentControlSource.kt +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/control/TorrentControlSource.kt @@ -90,6 +90,19 @@ internal interface TorrentControlSource { remove: List, ): List>? = null + /** + * Moves what the stopped task [taskId] saved at [from] to [to] and answers [sourceState] + * saving there, for a run that checks the files where they are now. + * + * @throws com.linroid.ketch.api.KetchError.Disk when the files cannot be moved + */ + suspend fun relocate( + taskId: String, + from: String, + to: String, + sourceState: SourceState?, + ): SourceState? = throw UnsupportedOperationException("Moving files is unavailable") + /** [state], as a task saved it, announcing to [tiers] in place of the torrent's trackers. */ fun withTrackers(state: SourceState, tiers: List>): SourceState = state diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/engine/KotlinTorrentEngine.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/engine/KotlinTorrentEngine.kt index 073f32592..408a83022 100644 --- a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/engine/KotlinTorrentEngine.kt +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/engine/KotlinTorrentEngine.kt @@ -1035,7 +1035,8 @@ internal class KotlinTorrentEngine( val requested = torrentSystemFileSystem.canonicalize(".".toPath()) .resolve(spec.outputPath).normalized() val store = TorrentPieceStore(spec.metadata, requested, spec.selected, spec.taskId, - fileSystem = payloadFileSystem, storageSlots = storageSlots, seedOnly = spec.seedOnly) + fileSystem = payloadFileSystem, storageSlots = storageSlots, seedOnly = spec.seedOnly, + adoptExisting = spec.adoptExisting) val output = store.outputPath.toPath() requireAvailableOutput(output) val checkpoint = spec.resumeData?.let(TorrentCheckpoint::decode) diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/engine/TorrentEngine.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/engine/TorrentEngine.kt index 9ff9fcfa2..1cedc6746 100644 --- a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/engine/TorrentEngine.kt +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/engine/TorrentEngine.kt @@ -135,4 +135,6 @@ internal data class TorrentTaskSpec( val seedOnly: Boolean = false, /** Reports the session's peers among the task's live connections. */ val connections: ConnectionReporter = ConnectionReporter.None, + /** Takes the files already at the task's paths as its own; see [TorrentPieceStore]. */ + val adoptExisting: Boolean = false, ) diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentMove.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentMove.kt new file mode 100644 index 000000000..19808cdd4 --- /dev/null +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentMove.kt @@ -0,0 +1,38 @@ +package com.linroid.ketch.torrent.storage + +import okio.FileSystem +import okio.IOException +import okio.Path + +/** + * Moves the file or folder [source] to [target], which must not exist: by renaming it, or + * where that fails, as between volumes, by copying it and then deleting the original. A copy + * that fails is deleted again, leaving [source] as it was. + */ +internal fun moveTorrentTree(files: FileSystem, source: Path, target: Path) { + check(files.metadataOrNull(target) == null) { "The folder already has something of that name" } + try { + files.atomicMove(source, target) + return + } catch (_: IOException) { + // Not on one volume: copied below. + } + try { + val below = if (files.metadata(source).isDirectory) files.listRecursively(source) else + emptySequence() + val paths = sequenceOf(source) + below + for (path in paths) { + val copy = if (path == source) target else target / path.relativeTo(source) + val info = files.metadata(path) + when { + info.symlinkTarget != null -> throw IOException("Cannot move a link") + info.isDirectory -> files.createDirectories(copy) + else -> files.copy(path, copy) + } + } + } catch (e: IOException) { + files.deleteRecursively(target, mustExist = false) + throw e + } + files.deleteRecursively(source, mustExist = false) +} diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentPieceStore.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentPieceStore.kt index 4ae03bd2b..d9b32f51c 100644 --- a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentPieceStore.kt +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentPieceStore.kt @@ -74,6 +74,12 @@ internal class TorrentPieceStore( private val fileSystem: FileSystem = torrentFileSystem, private val storageSlots: Semaphore = Semaphore(32), seedOnly: Boolean = false, + /** + * Takes the selected files already at their paths, and the folders holding them, as this + * task's own, as a task whose files were moved there or that was added to use them needs: + * cleanup deletes only what the task owns. + */ + private val adoptExisting: Boolean = false, ) { /** * Only seeds: while the selection is complete, a wanted piece that cannot be read back, or no @@ -256,6 +262,13 @@ internal class TorrentPieceStore( it.flush() } restamp(index) + } else if (adoptExisting) { + recordOwned(path, directory = false) + var folder = path.parent + while (folder != null && generateSequence(folder) { it.parent }.any { it == output }) { + recordOwned(folder, directory = true) + folder = folder.parent + } } } // Files of an earlier selection stay in place while this task still owns them. diff --git a/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentDownloadSourceTest.kt b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentDownloadSourceTest.kt index 062d09889..5c5395de0 100644 --- a/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentDownloadSourceTest.kt +++ b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentDownloadSourceTest.kt @@ -250,7 +250,7 @@ class TorrentDownloadSourceTest { val engine = setOf("file-selection", "v1", "v2", "hybrid", "utp", "mse-pe", "reachability", "settings", "seed-goals", "sequential", "export", "trackers", "recheck", "import", "file-priority", - "ip-filter", "tracker-edit") + "ip-filter", "tracker-edit", "relocate") // Local discovery only where the platform can join the LAN group. assertEquals(if (lsdSupported) engine + "local-discovery" else engine, source.torrentCapabilities) diff --git a/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentSeedingControlTest.kt b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentSeedingControlTest.kt index 9989a7491..e776a7ed3 100644 --- a/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentSeedingControlTest.kt +++ b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentSeedingControlTest.kt @@ -3,6 +3,7 @@ package com.linroid.ketch.torrent import com.linroid.ketch.api.Destination import com.linroid.ketch.api.DownloadRequest import com.linroid.ketch.api.DownloadState +import com.linroid.ketch.api.SpeedLimit import com.linroid.ketch.api.DownloadTask import com.linroid.ketch.core.Ketch import com.linroid.ketch.core.engine.HttpEngine @@ -295,6 +296,117 @@ class TorrentSeedingControlTest { source.assertNoLeaks() } + @Test + fun relocate_completedV1Task_movesItsFilesChecksThemAndOwnsThemThere() = harness { + val source = source(TorrentUploadPolicy.DISABLED) + val store = MemoryTaskStore() + val ketch = ketch(source, store) + ketch.start() + val task = ketch.download(request(0, 1)) + val first = task.awaitCompleted() + val controller = assertNotNull(ketch.controller(TorrentController)) + val folder = root / "moved" + + controller.relocate(task.taskId, folder.toString()) + + awaitStage("the check after the move", task) { + task.state.first { it is DownloadState.Completed && it.completedAt != first.completedAt } + } + val moved = folder / output.name + assertEquals(moved.toString(), task.outputPath) + assertEquals(moved.toString(), assertIs(task.state.value).outputPath) + assertEquals(moved.toString(), store.load(task.taskId)?.outputPath) + assertContentEquals(torrent.payloads[0], + torrentFileSystem.read(moved / "f0") { readByteArray() }) + assertContentEquals(torrent.payloads[1], + torrentFileSystem.read(moved / "f1") { readByteArray() }) + // Nothing stays where the files were, the task's hidden bookkeeping included. + assertNull(torrentFileSystem.metadataOrNull(output)) + assertTrue(torrentFileSystem.list(root).none { it.name.startsWith(".ketch-") }) + + // Where it is already, inside itself and onto something of its name are all refused. + torrentFileSystem.createDirectories(root / "taken" / output.name) + for (refused in listOf(folder, moved / "inner", root / "taken")) { + val error = assertFailsWith { + controller.relocate(task.taskId, refused.toString()) + } + assertEquals(TorrentCommandError.INVALID_INPUT, error.error) + } + assertEquals(moved.toString(), task.outputPath) + awaitStage("the task to be completed still", task) { + task.state.first { it is DownloadState.Completed } + } + + // The files are the task's own where they are now: removing it with its files deletes them. + task.remove(deleteFiles = true) + eventually("the moved files to be deleted") { + torrentFileSystem.metadataOrNull(moved / "f0") == null + } + source.assertNoLeaks() + } + + @Test + fun relocate_completedV2Task_movesItsFilesAndOwnsThemThere() = harness { + val store = MemoryTaskStore() + val record = v2Record("v2-move", v2Fixture, root / "v2").let { + it.copy(control = TaskControl(sharing = false)) + } + store.save(record) + val source = source(TorrentUploadPolicy.DISABLED) + val ketch = ketch(source, store) + ketch.start() + val task = ketch.tasks.value.single() + val first = assertIs(task.state.value) + val controller = assertNotNull(ketch.controller(TorrentController)) + val folder = root / "elsewhere" + + controller.relocate(task.taskId, folder.toString()) + + awaitStage("the check after the move", task) { + task.state.first { it is DownloadState.Completed && it.completedAt != first.completedAt } + } + val moved = folder / "v2" + assertEquals(moved.toString(), task.outputPath) + assertContentEquals(v2Fixture.payloads[0], + torrentFileSystem.read(moved / "a") { readByteArray() }) + assertNull(torrentFileSystem.metadataOrNull(root / "v2")) + assertTrue(torrentFileSystem.list(root).none { it.name.startsWith(".ketch-") }) + assertTrue(source.decodeState(assertNotNull(store.load(task.taskId)?.sourceState)).relocated) + + task.remove(deleteFiles = true) + eventually("the moved files to be deleted") { + torrentFileSystem.metadataOrNull(moved / "a") == null + } + source.assertNoLeaks() + } + + @Test + fun relocate_downloadingTask_goesOnAtTheNewPlace() = harness { + val source = source(TorrentUploadPolicy.DISABLED) + val ketch = ketch(source) + ketch.start() + // Slow enough to still be downloading when it is moved. + val task = ketch.download(request(0, 1).copy(speedLimit = SpeedLimit.of(2048))) + eventually("the download to be under way") { + task.outputPath != null && task.state.value is DownloadState.Downloading && + torrentFileSystem.metadataOrNull(output / "f0") != null + } + val folder = root / "later" + + assertNotNull(ketch.controller(TorrentController)).relocate(task.taskId, folder.toString()) + + task.setSpeedLimit(SpeedLimit.Unlimited) + task.awaitCompleted() + val moved = folder / output.name + assertEquals(moved.toString(), task.outputPath) + assertContentEquals(torrent.payloads[0], + torrentFileSystem.read(moved / "f0") { readByteArray() }) + assertContentEquals(torrent.payloads[1], + torrentFileSystem.read(moved / "f1") { readByteArray() }) + assertNull(torrentFileSystem.metadataOrNull(output)) + source.assertNoLeaks() + } + @Test fun recheck_taskItsUserStoppedSeeding_staysStopped() = harness { val source = source() diff --git a/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/storage/TorrentMoveTest.kt b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/storage/TorrentMoveTest.kt new file mode 100644 index 000000000..a338a4bc9 --- /dev/null +++ b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/storage/TorrentMoveTest.kt @@ -0,0 +1,79 @@ +package com.linroid.ketch.torrent.storage + +import com.linroid.ketch.torrent.crypto.torrentRandomBytes +import com.linroid.ketch.torrent.metainfo.InfoHash +import okio.FileSystem +import okio.ForwardingFileSystem +import okio.IOException +import okio.Path +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull + +class TorrentMoveTest { + /** A file system that cannot rename, as between two volumes, and may fail to copy [failing]. */ + private class NoRename(delegate: FileSystem, val failing: String? = null) : + ForwardingFileSystem(delegate) { + override fun atomicMove(source: Path, target: Path): Unit = throw IOException("cross-device") + + override fun copy(source: Path, target: Path) { + if (source.name == failing) throw IOException("disk full") + super.copy(source, target) + } + } + + private fun tree(block: (root: Path) -> Unit) { + val root = FileSystem.SYSTEM_TEMPORARY_DIRECTORY / + "ketch-move-${InfoHash.fromBytes(torrentRandomBytes(20)).hex}" + try { + torrentFileSystem.createDirectories(root / "from" / "pack" / "inner") + torrentFileSystem.write(root / "from" / "pack" / "a.bin") { write(byteArrayOf(1, 2, 3)) } + torrentFileSystem.write(root / "from" / "pack" / "inner" / "b.bin") { write(ByteArray(0)) } + torrentFileSystem.createDirectories(root / "to") + block(root) + } finally { + torrentFileSystem.deleteRecursively(root, mustExist = false) + } + } + + private fun read(path: Path) = torrentFileSystem.read(path) { readByteArray() } + + @Test + fun `a tree is renamed where it can be and copied where it cannot`() = tree { root -> + moveTorrentTree(torrentFileSystem, root / "from" / "pack", root / "to" / "pack") + assertContentEquals(byteArrayOf(1, 2, 3), read(root / "to" / "pack" / "a.bin")) + assertNull(torrentFileSystem.metadataOrNull(root / "from" / "pack")) + + moveTorrentTree(NoRename(torrentFileSystem), root / "to" / "pack", root / "from" / "pack") + assertContentEquals(byteArrayOf(1, 2, 3), read(root / "from" / "pack" / "a.bin")) + assertEquals(0L, torrentFileSystem.metadata(root / "from" / "pack" / "inner" / "b.bin").size) + assertNull(torrentFileSystem.metadataOrNull(root / "to" / "pack")) + + // A single file moves the same way. + moveTorrentTree(NoRename(torrentFileSystem), root / "from" / "pack" / "a.bin", + root / "to" / "a.bin") + assertContentEquals(byteArrayOf(1, 2, 3), read(root / "to" / "a.bin")) + assertNull(torrentFileSystem.metadataOrNull(root / "from" / "pack" / "a.bin")) + } + + @Test + fun `a copy that fails leaves the files where they were`() = tree { root -> + assertFailsWith { + moveTorrentTree(NoRename(torrentFileSystem, failing = "b.bin"), root / "from" / "pack", + root / "to" / "pack") + } + assertContentEquals(byteArrayOf(1, 2, 3), read(root / "from" / "pack" / "a.bin")) + assertNull(torrentFileSystem.metadataOrNull(root / "to" / "pack")) + } + + @Test + fun `nothing is moved onto something that is there`() = tree { root -> + torrentFileSystem.createDirectories(root / "to" / "pack") + assertFailsWith { + moveTorrentTree(torrentFileSystem, root / "from" / "pack", root / "to" / "pack") + } + assertContentEquals(byteArrayOf(1, 2, 3), read(root / "from" / "pack" / "a.bin")) + } +} From aa9fd472f8978745b8d384beea2d7571403819c9 Mon Sep 17 00:00:00 2001 From: Lin Zhang Date: Sat, 14 Nov 2026 19:07:55 +0900 Subject: [PATCH 2/3] style(server): wrap a long test line --- .../test/kotlin/com/linroid/ketch/server/TorrentRoutesTest.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/library/server/src/test/kotlin/com/linroid/ketch/server/TorrentRoutesTest.kt b/library/server/src/test/kotlin/com/linroid/ketch/server/TorrentRoutesTest.kt index 92e3267c0..b38da1e26 100644 --- a/library/server/src/test/kotlin/com/linroid/ketch/server/TorrentRoutesTest.kt +++ b/library/server/src/test/kotlin/com/linroid/ketch/server/TorrentRoutesTest.kt @@ -485,7 +485,8 @@ class TorrentRoutesTest { val edited = client.put("/api/torrents/t1/trackers") { contentType(ContentType.Application.Json) - setBody("""{"add":["udp://t.example:6969/announce"],"remove":["udp://old.example:1"]}""") + setBody(json.encodeToString(TorrentTrackerEdit(listOf("udp://t.example:6969/announce"), + listOf("udp://old.example:1")))) } val refused = client.put("/api/torrents/t1/trackers") { contentType(ContentType.Application.Json) From fc27c8c501c2c3ff765b6bbb02ac8ad2a7ac0998 Mon Sep 17 00:00:00 2001 From: Lin Zhang Date: Sat, 14 Nov 2026 22:54:29 +0900 Subject: [PATCH 3/3] fix(torrent): guard the whole move, keep ownership before the next run and finish a copied move --- AGENTS.md | 6 +++- .../linroid/ketch/server/DestinationGuard.kt | 22 ++++++++++---- .../linroid/ketch/server/api/TorrentRoutes.kt | 2 +- .../ketch/torrent/TorrentDownloadSource.kt | 30 ++++++++++++++----- .../ketch/torrent/storage/TorrentMove.kt | 15 +++++++--- .../torrent/TorrentSeedingControlTest.kt | 30 +++++++++++++++++++ .../ketch/torrent/storage/TorrentMoveTest.kt | 24 +++++++++++++-- 7 files changed, 108 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a7d080bf3..2c1e45c6e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1381,7 +1381,11 @@ docker/ # Docker image of `ketch server` (Dockerfile, entrypoint, compose seed again (`prepareRecheck`, `SourceHost.reopen`); a paused one is checked when it next starts. Payload counters start again with the new checkpoint, and pieces kept for skipped files in the v1 sidecar are downloaded again when their files are chosen. The server lets a - caller move only where it may save (`DestinationGuard.checkMove`, 403 `path_rejected`). The + caller move only files it may delete, into a folder and to a path (`/`) it + may save at (`DestinationGuard.checkMove`, 403 `path_rejected`). A copy between volumes whose + original cannot be deleted afterwards still counts as moved, with a warning, and a moved task + removed with its files before it runs again takes them as its own first (`v1Store` and + `v2Store` with `adopt`). The apps have no control for it yet - Editing trackers: `TorrentController.editTrackers(taskId, TorrentTrackerEdit(add, remove))` (REST `PUT /api/torrents/{id}/trackers`, answering the list; capability `tracker-edit`) removes diff --git a/library/server/src/main/kotlin/com/linroid/ketch/server/DestinationGuard.kt b/library/server/src/main/kotlin/com/linroid/ketch/server/DestinationGuard.kt index 9031fd1dd..3f3172ffc 100644 --- a/library/server/src/main/kotlin/com/linroid/ketch/server/DestinationGuard.kt +++ b/library/server/src/main/kotlin/com/linroid/ketch/server/DestinationGuard.kt @@ -164,16 +164,26 @@ internal class DestinationGuard( } /** - * Checks that a download's files may be moved into [directory], as a download may be saved - * there. + * Checks that [task]'s files may be moved into [directory] (only the folder for a task this + * server does not have, which no move then happens for): that they may be deleted where + * they are ([checkDeletion]), as a move takes them away from there, and that both the folder + * and the path they get in it, the folder and the name they have, are where a download may be + * saved, so no move makes a hidden or protected file out of a download's name. * - * @throws PathRejectedException if the folder lies outside the folders + * @throws PathRejectedException if the files, the folder or the new path lie outside the + * folders */ - suspend fun checkMove(directory: String) { - val status = ketch.status() - if (passesAsUri(directory) || !policy(status.system.downloadDirectory).contains(directory)) { + suspend fun checkMove(task: DownloadTask?, directory: String) { + task?.let { checkDeletion(it) } + val policy = policy(ketch.status().system.downloadDirectory) + if (passesAsUri(directory) || !policy.contains(directory)) { throw PathRejectedException(directory) } + val from = task?.outputPath ?: (task?.state?.value as? DownloadState.Completed)?.outputPath + val name = from?.trimEnd('/', '\\')?.substringAfterLast('/')?.substringAfterLast('\\') + if (name.isNullOrEmpty()) return + val target = directory.trimEnd('/', '\\') + "/" + name + if (!policy.contains(target)) throw PathRejectedException(target) } /** diff --git a/library/server/src/main/kotlin/com/linroid/ketch/server/api/TorrentRoutes.kt b/library/server/src/main/kotlin/com/linroid/ketch/server/api/TorrentRoutes.kt index 66ae8e8eb..4c4fc2b15 100644 --- a/library/server/src/main/kotlin/com/linroid/ketch/server/api/TorrentRoutes.kt +++ b/library/server/src/main/kotlin/com/linroid/ketch/server/api/TorrentRoutes.kt @@ -162,7 +162,7 @@ internal fun Route.torrentRoutes( val taskId = resource.parent.id val body = call.receiveJson() log.d { "PUT /api/torrents/$taskId/location" } - destinations.checkMove(body.directory) + destinations.checkMove(ketch.tasks.value.firstOrNull { it.taskId == taskId }, body.directory) controller.relocate(taskId, body.directory) call.respond(controller.snapshot(taskId) ?: throw notFound(taskId)) } diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentDownloadSource.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentDownloadSource.kt index afc329e82..a0f0c183e 100644 --- a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentDownloadSource.kt +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/TorrentDownloadSource.kt @@ -499,7 +499,10 @@ class TorrentDownloadSource( try { target.parent?.let { files.createDirectories(it) } // A task that created nothing yet has nothing to move. - if (files.metadataOrNull(source) != null) moveTorrentTree(files, source, target) + if (files.metadataOrNull(source) != null && !moveTorrentTree(files, source, target)) { + // The task saves at its new place all the same: the copy there is whole. + log.w { "Moved taskId=$taskId, but could not delete all of what it left behind" } + } } catch (e: IOException) { throw KetchError.Disk(e) } @@ -2511,7 +2514,7 @@ class TorrentDownloadSource( engine.load()?.removeTorrent(state.infoHash, deleteFiles = true) } tasks.release(context.taskId) - val store = v1Store(context, state) ?: return + val store = v1Store(context, state, adopt = true) ?: return store.cleanup() stateMutex.withLock { states.remove(context.taskId) } } catch (e: CancellationException) { throw e @@ -2543,10 +2546,15 @@ class TorrentDownloadSource( } } - /** [state]'s v1 store with the files it owns recovered, or null when nothing was saved. */ + /** + * [state]'s v1 store with the files it owns recovered, or null when nothing was saved. With + * [adopt], a task whose files were moved and that has not run since takes the files at its + * new place as its own first, as its next run would have: nothing else records that they are. + */ private suspend fun v1Store( context: DownloadContext, state: TorrentSourceState, + adopt: Boolean = false, ): TorrentPieceStore? { val checkpoint = state.resumeData.takeIf { it.isNotEmpty() } ?.let { TorrentCheckpoint.decode(decodeBase64(it)) } @@ -2555,15 +2563,17 @@ class TorrentDownloadSource( ?: return null val output = context.outputPath ?: checkpoint?.output ?: state.savePath if (output.isEmpty()) return null + val adopting = adopt && state.relocated && checkpoint == null val store = TorrentPieceStore(metadata, output.toPath(), - state.selectedFileIds.map { it.toInt() }.toSet(), context.taskId) + state.selectedFileIds.map { it.toInt() }.toSet(), context.taskId, + adoptExisting = adopting) checkpoint?.let { store.restore(it) } - store.recoverOwnership() + if (adopting) store.initialize() else store.recoverOwnership() return store } private suspend fun cleanupV2(context: DownloadContext, state: TorrentSourceState) { - val store = v2Store(context, state) + val store = v2Store(context, state, adopt = true) try { store.cleanup() stateMutex.withLock { states.remove(context.taskId) } @@ -2574,6 +2584,7 @@ class TorrentDownloadSource( private suspend fun v2Store( context: DownloadContext, state: TorrentSourceState, + adopt: Boolean = false, ): TorrentV2PieceStore { check(!tasks.isReserved(context.taskId)) { "Join the torrent download before cleanup" } val metainfo = requireNotNull(metainfoOf(context.preResolved, state)) { "No saved metainfo" } @@ -2588,6 +2599,7 @@ class TorrentDownloadSource( val checkpoint = state.resumeData.takeIf { it.isNotEmpty() }?.let { requireNotNull(TorrentV2Checkpoint.decode(decodeBase64(it))) } + val adopting = adopt && state.relocated && checkpoint == null // The creation log may be bound to any selection the task ran with: the checkpoint's, the // one the state mirrors, or the record's. val store = TorrentV2PieceStore(document, absolute, selected, context.taskId, @@ -2595,10 +2607,14 @@ class TorrentDownloadSource( Semaphore(config.maxOpenPayloadFiles), creationLogPath = v2CreationLog(absolute, context.taskId), legacySelections = listOfNotNull(checkpoint?.selected, state.selectedFileIds, - context.request.selectedFileIds).filter { it.isNotEmpty() }) + context.request.selectedFileIds).filter { it.isNotEmpty() }, + // A task whose files were moved and that has not run since owns them only once a store + // takes them, as its next run would have. + adoptExisting = adopting) try { checkpoint?.let { store.restore(it) } store.recoverOwnership() + if (adopting) store.initialize() } catch (e: Throwable) { store.close() throw e diff --git a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentMove.kt b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentMove.kt index 19808cdd4..769b1ef35 100644 --- a/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentMove.kt +++ b/library/torrent/src/commonMain/kotlin/com/linroid/ketch/torrent/storage/TorrentMove.kt @@ -7,13 +7,15 @@ import okio.Path /** * Moves the file or folder [source] to [target], which must not exist: by renaming it, or * where that fails, as between volumes, by copying it and then deleting the original. A copy - * that fails is deleted again, leaving [source] as it was. + * that fails is deleted again, leaving [source] as it was. Once the copy is complete the files + * are at [target] for good: returns false when the original could then not be deleted whole, + * and what is left of it stays where it was. */ -internal fun moveTorrentTree(files: FileSystem, source: Path, target: Path) { +internal fun moveTorrentTree(files: FileSystem, source: Path, target: Path): Boolean { check(files.metadataOrNull(target) == null) { "The folder already has something of that name" } try { files.atomicMove(source, target) - return + return true } catch (_: IOException) { // Not on one volume: copied below. } @@ -34,5 +36,10 @@ internal fun moveTorrentTree(files: FileSystem, source: Path, target: Path) { files.deleteRecursively(target, mustExist = false) throw e } - files.deleteRecursively(source, mustExist = false) + return try { + files.deleteRecursively(source, mustExist = false) + true + } catch (_: IOException) { + false + } } diff --git a/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentSeedingControlTest.kt b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentSeedingControlTest.kt index e776a7ed3..ef4338b7b 100644 --- a/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentSeedingControlTest.kt +++ b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/TorrentSeedingControlTest.kt @@ -39,6 +39,7 @@ import kotlinx.coroutines.withTimeout import kotlinx.serialization.json.Json import okio.FileSystem import okio.Path +import okio.Path.Companion.toPath import kotlin.concurrent.atomics.AtomicInt import kotlin.concurrent.atomics.ExperimentalAtomicApi import kotlin.concurrent.atomics.incrementAndFetch @@ -380,6 +381,35 @@ class TorrentSeedingControlTest { source.assertNoLeaks() } + @Test + fun relocate_taskRemovedWithItsFilesBeforeItRunsAgain_deletesThemAtTheNewPlace() = harness { + val source = source(TorrentUploadPolicy.DISABLED) + val store = MemoryTaskStore() + // Moved while they were not running, as a paused task is: nothing has checked them since. + val v1 = v1Record("v1-moved", MultiFileTorrent(listOf(40_000, 30_000)), root / "one", + sharing = false) + val v2 = v2Record("v2-moved", v2Fixture, root / "two") + .let { it.copy(control = TaskControl(sharing = false)) } + for (record in listOf(v1, v2)) { + val from = checkNotNull(record.outputPath) + val to = (root / "moved" / from.toPath().name).toString() + val state = source.relocate(record.taskId, from, to, record.sourceState) + store.save(record.copy(outputPath = to, sourceState = state)) + } + val ketch = ketch(source, store) + ketch.start() + assertTrue(torrentFileSystem.exists(root / "moved" / "one" / "f0")) + assertTrue(torrentFileSystem.exists(root / "moved" / "two" / "a")) + + for (task in ketch.tasks.value) task.remove(deleteFiles = true) + + eventually("the moved files to be deleted") { + torrentFileSystem.metadataOrNull(root / "moved" / "one" / "f0") == null && + torrentFileSystem.metadataOrNull(root / "moved" / "two" / "a") == null + } + source.assertNoLeaks() + } + @Test fun relocate_downloadingTask_goesOnAtTheNewPlace() = harness { val source = source(TorrentUploadPolicy.DISABLED) diff --git a/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/storage/TorrentMoveTest.kt b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/storage/TorrentMoveTest.kt index a338a4bc9..10fc2226d 100644 --- a/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/storage/TorrentMoveTest.kt +++ b/library/torrent/src/commonTest/kotlin/com/linroid/ketch/torrent/storage/TorrentMoveTest.kt @@ -10,12 +10,22 @@ import kotlin.test.Test import kotlin.test.assertContentEquals import kotlin.test.assertEquals import kotlin.test.assertFailsWith +import kotlin.test.assertFalse import kotlin.test.assertNull +import kotlin.test.assertTrue class TorrentMoveTest { /** A file system that cannot rename, as between two volumes, and may fail to copy [failing]. */ - private class NoRename(delegate: FileSystem, val failing: String? = null) : - ForwardingFileSystem(delegate) { + private class NoRename( + delegate: FileSystem, + val failing: String? = null, + val keeps: Boolean = false, + ) : ForwardingFileSystem(delegate) { + override fun delete(path: Path, mustExist: Boolean) { + if (keeps) throw IOException("read-only") + super.delete(path, mustExist) + } + override fun atomicMove(source: Path, target: Path): Unit = throw IOException("cross-device") override fun copy(source: Path, target: Path) { @@ -68,6 +78,16 @@ class TorrentMoveTest { assertNull(torrentFileSystem.metadataOrNull(root / "to" / "pack")) } + @Test + fun `a copy whose original cannot be deleted still counts as moved`() = tree { root -> + // The files are whole at their new place, so that is where the task saves from now on. + assertFalse(moveTorrentTree(NoRename(torrentFileSystem, keeps = true), root / "from" / "pack", + root / "to" / "pack")) + assertContentEquals(byteArrayOf(1, 2, 3), read(root / "to" / "pack" / "a.bin")) + assertContentEquals(byteArrayOf(1, 2, 3), read(root / "from" / "pack" / "a.bin")) + assertTrue(moveTorrentTree(torrentFileSystem, root / "to" / "pack", root / "to" / "again")) + } + @Test fun `nothing is moved onto something that is there`() = tree { root -> torrentFileSystem.createDirectories(root / "to" / "pack")