From c0ebc048d893e5d23c224bdd62883199c1fa19d2 Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:54:44 +0000 Subject: [PATCH] Rank related and trending projects by how long people play, and count visit depth Related projects now lead with projects that at least three of the same players stayed in, on projects with at least five such players, weighted toward players with fewer plays. Co-played projects fill at most half the list, before the same owner, shared tags and trending. Trending and the daily featured project weight week views by the share of signed-in players with 30 seconds or more of total playtime, smoothed so projects with little playtime sit in the middle. Only plays updated in the last 90 days count. Owners' own plays, banned and shadowbanned players, and players whose recent activity is hidden (minors and students by default) never count, so co-play only draws on activity their profiles already show. The signals are rebuilt from project_playtime under one lock and cached for five minutes, and ties sort by a single comparator so the order is stable. The funnel accepts visit_start (new or returning device) and visit_depth (2, 3, 5 or 10 projects in one visit). --- opal.json | 2 +- src/admin/admin.osl | 5 + src/admin/funnel-events-policy.osl | 2 + src/admin/funnel-events-policy.test.osl | 3 + src/git/pulls.osl | 1 + src/projects/play-signals.osl | 143 ++++++++++++++++++++++++ src/projects/play-signals.test.osl | 66 +++++++++++ src/projects/project-discovery.osl | 60 +++++++++- src/projects/related-projects.osl | 96 ++++++++++------ src/projects/related-projects.test.osl | 16 ++- src/storage/storage.osl | 5 + 11 files changed, 352 insertions(+), 47 deletions(-) create mode 100644 src/projects/play-signals.osl create mode 100644 src/projects/play-signals.test.osl diff --git a/opal.json b/opal.json index be3259a..bbbb63e 100644 --- a/opal.json +++ b/opal.json @@ -1,6 +1,6 @@ { "name": "mistwarp-api", - "version": "1.0.1", + "version": "1.1.0", "main": "main.osl", "scripts": { "dev": ["osl", "run", "main.osl"], diff --git a/src/admin/admin.osl b/src/admin/admin.osl index 8819839..3e1574c 100644 --- a/src/admin/admin.osl +++ b/src/admin/admin.osl @@ -7,6 +7,7 @@ export { loadBans, projectHiddenFrom, projectOwnerBanned, + playerUnbanned, projectsVisibleTo, projectsVisibleToEveryone, saveAdmins, @@ -103,6 +104,10 @@ def projectsVisibleTo(array entries,*serve.Context c) array ( return projectEntriesListed(entries, bannedOwners(), shadowBannedSet(), normalizeUsername(contextString(c, "username")), contextString(c, "userId"), contextBool(c, "isAdmin")) ) +def playerUnbanned(string username, string userId) boolean ( + return !ownerInBannedSet(bannedOwners(), username, userId) and !ownerInBannedSet(shadowBannedSet(), username, userId) +) + def projectsVisibleToEveryone(array entries) array ( return projectEntriesListed(entries, bannedOwners(), shadowBannedSet(), "", "", false) ) diff --git a/src/admin/funnel-events-policy.osl b/src/admin/funnel-events-policy.osl index 36eeba8..d18bfb5 100644 --- a/src/admin/funnel-events-policy.osl +++ b/src/admin/funnel-events-policy.osl @@ -4,6 +4,8 @@ object funnelEventVariants = { editor_open: [], home_view: [], project_view: ["signed_in", "signed_out"], + visit_start: ["new", "returning"], + visit_depth: ["2", "3", "5", "10"], project_play_start: [], start_creating: ["home"], signin_prompt_shown: ["remix", "save", "library", "react", "comment", "follow"], diff --git a/src/admin/funnel-events-policy.test.osl b/src/admin/funnel-events-policy.test.osl index d91e28c..8df8581 100644 --- a/src/admin/funnel-events-policy.test.osl +++ b/src/admin/funnel-events-policy.test.osl @@ -13,6 +13,9 @@ testing.assert(funnelEventKey("made_up", "") == "", "unknown events are rejected testing.assert(funnelEventKey("", "remix") == "", "empty names are rejected") testing.assert(funnelEventKey("share_opened", "update") == "share_opened:update", "share opens count by action") testing.assert(funnelEventKey("starter_opened", "clicker") == "starter_opened:clicker", "starter projects count by id") +testing.assert(funnelEventKey("visit_start", "returning") == "visit_start:returning", "visits count new and returning devices") +testing.assert(funnelEventKey("visit_depth", "5") == "visit_depth:5", "visits count how many projects they reached") +testing.assert(funnelEventKey("visit_depth", "4") == "visit_depth", "unlisted depths fall back to the event name") testing.assert(isFunnelOrigin("https://mistwarp.org"), "the community site can count") testing.assert(isFunnelOrigin("https://editor.mistwarp.org"), "the editor origin can count") testing.assert(!isFunnelOrigin("http://mistwarp.org"), "plain http is rejected") diff --git a/src/git/pulls.osl b/src/git/pulls.osl index 614c3d3..6a64433 100644 --- a/src/git/pulls.osl +++ b/src/git/pulls.osl @@ -897,6 +897,7 @@ def fastForwardPullLocked(*serve.Context c, object pull, object target, object s target.assets = source.assets target.sizeBytes = projectSizeBytes(source) target.edited = timestamp + target.lastUploadAt = timestamp if source.contains("jsonBytes") ( target.jsonBytes = source.jsonBytes ) diff --git a/src/projects/play-signals.osl b/src/projects/play-signals.osl new file mode 100644 index 0000000..0b749a2 --- /dev/null +++ b/src/projects/play-signals.osl @@ -0,0 +1,143 @@ +export {} + +number heldPlayMs = 30000 +number playWindowMs = 7776000000 +number minHeldPlayers = 5 +number minCoPlayers = 3 + +class PlaySignals ( + string[number] players = {} + string[number] held = {} + string[string[]] heldBy = {} + string[string[]] heldProjects = {} +) + +struct RankedId ( + string id = "" + number score = 0 +) + +struct RankedEntry ( + number index = 0 + number rank = 0 + number views = 0 +) + +def playerKeyOf(string userId, string username) string ( + if userId.strip() != "" ( + return userId.strip() + ) + return username.toLower().strip() +) + +def addHeldPlay(PlaySignals signals, string projectId, string player) ( + signals.held[projectId] = (signals.held[projectId] ?? 0) + 1 + string[] byProject = signals.heldBy[projectId] ?? [] + byProject.append(player) + signals.heldBy[projectId] = byProject + string[] byPlayer = signals.heldProjects[player] ?? [] + byPlayer.append(projectId) + signals.heldProjects[player] = byPlayer +) + +def playSignalsFrom(object[] rows, string[string] ownerKeys, number now) PlaySignals ( + PlaySignals signals = PlaySignals() + for i rows.len ( + object row = rows[i] + string projectId = (row.projectId ?? "").toStr() + string username = (row.username ?? "").toStr() + string player = playerKeyOf((row.userId ?? "").toStr(), username) + string owner = ownerKeys[projectId] ?? "" + boolean ownPlay = owner != "" and username.toLower().strip() == owner + boolean recent = now - (row.updated ?? 0).toNum() <= playWindowMs + if projectId != "" and player != "" and !ownPlay and recent ( + signals.players[projectId] = (signals.players[projectId] ?? 0) + 1 + if (row.duration ?? 0).toNum() >= heldPlayMs ( + addHeldPlay(signals, projectId, player) + ) + ) + ) + return signals +) + +def holdWeight(PlaySignals signals, string projectId) number ( + number playerCount = signals.players[projectId] ?? 0 + number heldCount = signals.held[projectId] ?? 0 + return (heldCount + 2) / (playerCount + 4) +) + +def compareRankedEntries(RankedEntry a, RankedEntry b) int ( + if a.rank != b.rank ( + return if a.rank > b.rank then -1 else 1 + ) + if a.views != b.views ( + return if a.views > b.views then -1 else 1 + ) + return (a.index - b.index).toInt() +) + +def trendingOrder(object[] entries, PlaySignals signals) object[] ( + RankedEntry[] ranked = [] + for i entries.len ( + object entry = entries[i] + number views = (entry.weekViews ?? 0).toNum() + ranked.append(RankedEntry(i, views * holdWeight(signals, entry.id.toStr()), views)) + ) + RankedEntry[] sorted = ranked.sort(compareRankedEntries) + object[] ordered = [] + for i sorted.len ( + ordered.append(entries[sorted[i].index]) + ) + return ordered +) + +def compareRankedIds(RankedId a, RankedId b) int ( + if a.score != b.score ( + return if a.score > b.score then -1 else 1 + ) + if a.id == b.id ( + return 0 + ) + return if a.id < b.id then -1 else 1 +) + +def coPlayScores(PlaySignals signals, string projectId) object ( + string[] sharedPlayers = signals.heldBy[projectId] ?? [] + string[number] scores = {} + string[number] counts = {} + for i sharedPlayers.len ( + string[] playedNext = signals.heldProjects[sharedPlayers[i]] ?? [] + number share = 1 / (playedNext.len.toNum().sqrt() ?? 1) + for j playedNext.len ( + string otherId = playedNext[j] + if otherId != projectId ( + scores[otherId] = (scores[otherId] ?? 0) + share + counts[otherId] = (counts[otherId] ?? 0) + 1 + ) + ) + ) + return {scores: scores, counts: counts} +) + +def coPlayedIds(PlaySignals signals, string projectId, number limit) string[] ( + string[] ids = [] + if (signals.held[projectId] ?? 0) < minHeldPlayers ( + return ids + ) + object found = coPlayScores(signals, projectId) + string[number] scores = found.scores.assert(string[number]) + string[number] counts = found.counts.assert(string[number]) + RankedId[] ranked = [] + for otherId of scores ( + if (counts[otherId] ?? 0) >= minCoPlayers ( + ranked.append(RankedId(otherId, scores[otherId] ?? 0)) + ) + ) + RankedId[] sorted = ranked.sort(compareRankedIds) + for i sorted.len ( + if ids.len < limit ( + ids.append(sorted[i].id) + ) + ) + return ids +) diff --git a/src/projects/play-signals.test.osl b/src/projects/play-signals.test.osl new file mode 100644 index 0000000..56f2610 --- /dev/null +++ b/src/projects/play-signals.test.osl @@ -0,0 +1,66 @@ +import "std:testing" +import "./play-signals.osl" + +number now = 8000000000000 + +def play(string projectId, string player, number duration) object ( + return {_id: projectId ++ ":" ++ player, projectId: projectId, userId: player, username: player, duration: duration, updated: now} +) + +def ids(string[] items) string ( + string res = "" + for i items.len ( + res ++= items[i] ++ " " + ) + return res.strip() +) + +def entryIds(object[] items) string ( + string res = "" + for i items.len ( + object item = items[i] + res ++= item.id.toStr() + ) + return res +) + +object[] rows = [] +for i 5 ( + string player = "fan" ++ i.toStr() + rows.append(play("bait", player, 60000)) + if i <= 3 ( + rows.append(play("pair", player, 60000)) + ) + if i <= 2 ( + rows.append(play("thin", player, 60000)) + ) +) +rows.append(play("bait", "maker", 900000)) +rows.append(play("solo", "maker", 900000)) +rows.append(play("bait", "skimmer", 2000)) +rows.append({_id: "bait:old", projectId: "bait", userId: "old", username: "old", duration: 60000, updated: now - 9000000000}) +rows.append({_id: "pair:legacy", projectId: "pair", userId: "", username: " Legacy ", duration: 60000, updated: now}) +string[string] owners = {bait: "maker", solo: "maker", pair: "someone", thin: "someone"} +PlaySignals signals = playSignalsFrom(rows, owners, now) + +testing.assert(holdWeight(signals, "bait") == 7 / 10, "held plays raise the weight, the owner's and old plays do not count") +testing.assert(holdWeight(signals, "unplayed") == 0.5, "projects nobody has played sit in the middle") +testing.assert(ids(coPlayedIds(signals, "bait", 5)) == "pair", "a project needs three shared players to count as co-played") +testing.assert(ids(coPlayedIds(signals, "thin", 5)) == "", "projects with few held players get no co-plays") +testing.assert(ids(coPlayedIds(signals, "solo", 5)) == "", "the owner's own plays never link projects") +testing.assert((signals.players["pair"] ?? 0) == 4, "legacy rows without a user id count once under their username") + +object[] tied = [] +for i 30 ( + tied.append({id: "t" ++ (100 + i).toStr(), weekViews: 5}) +) +testing.assert(entryIds(trendingOrder(tied, PlaySignals())).startsWith("t101t102t103t104t105t106t107t108t109t110t111t112t113t114t115"), "equal ranks keep their incoming order on long lists") + +object[] entries = [ + {id: "pair", weekViews: 10}, + {id: "bait", weekViews: 9}, + {id: "x", weekViews: 9}, + {id: "z", weekViews: 0} +] +testing.assert(entryIds(trendingOrder(entries, signals)) == "pairbaitxz", "views are weighted by how long plays last") +testing.assert(entryIds(trendingOrder(entries, PlaySignals())) == "pairbaitxz", "without playtime the order follows views") diff --git a/src/projects/project-discovery.osl b/src/projects/project-discovery.osl index a91284a..8a0566e 100644 --- a/src/projects/project-discovery.osl +++ b/src/projects/project-discovery.osl @@ -5,6 +5,9 @@ export { // osl-disable-next-line OSL-UNTYPED-GLOBAL auto sharedProjectsCache = cache.new({capacity: 2, ttl: 60000}) +// osl-disable-next-line OSL-UNTYPED-GLOBAL +auto playSignalsCache = cache.new({capacity: 2, ttl: 300000}) + def sharedIndexEntries() array ( if sharedProjectsCache.has("projects") ( return sharedProjectsCache.get("projects").assert(array) @@ -21,6 +24,51 @@ def sharedIndexEntries() array ( return entries ) +def sharedPlayRows(array rows) object[] ( + string[boolean] allowed = {} + object[] kept = [] + for i rows.len ( + object row = rows[i].assert(object) + string username = (row.username ?? "").toStr() + string key = normalizeUsername(username) + if key != "" and !allowed.contains(key) ( + allowed[key] = playerUnbanned(username, (row.userId ?? "").toStr()) and effectiveShowRecentActivity(loadProfile(username)) + ) + if key != "" and (allowed[key] ?? false) ( + kept.append(row) + ) + ) + return kept +) + +def buildPlaySignals() PlaySignals ( + string[string] ownerKeys = {} + array entries = sharedIndexEntries() + for i entries.len ( + object entry = entries[i] + ownerKeys[entry.id.toStr()] = (entry.ownerKey ?? entry.owner ?? "").toStr().toLower().strip() + ) + array rows = playtimeDocs.all() catch (err) ( + log "WARNING: could not read playtime for discovery: " ++ err.toStr() + return PlaySignals() + ) + return playSignalsFrom(sharedPlayRows(rows), ownerKeys, timestamp) +) + +def currentPlaySignals() PlaySignals ( + if playSignalsCache.has("signals") ( + return playSignalsCache.get("signals").assert(PlaySignals) + ) + sync.lock("play-signals") + defer sync.unlock("play-signals") + if playSignalsCache.has("signals") ( + return playSignalsCache.get("signals").assert(PlaySignals) + ) + PlaySignals signals = buildPlaySignals() + playSignalsCache.set("signals", signals) + return signals +) + def handleExplore(*serve.Context c) ( array entries = projectsVisibleTo(sharedIndexEntries(), c) @@ -87,7 +135,7 @@ def handleExplore(*serve.Context c) ( ) else if sort == "undiscovered" ( entries.sortBy("views", "ascending") ) else if sort == "trending" ( - entries.sortBy("weekViews", "descending") + entries = trendingOrder(entries, currentPlaySignals()) ) else ( entries.sortBy("sharedAt", "descending") ) @@ -109,15 +157,15 @@ def handleRandomProject(*serve.Context c) ( ) def handleRelatedProjects(*serve.Context c) ( - array entries = projectsVisibleToEveryone(sharedIndexEntries()) - entries.sortBy("weekViews", "descending") + PlaySignals signals = currentPlaySignals() + object[] entries = trendingOrder(projectsVisibleToEveryone(sharedIndexEntries()), signals) number limit = clampNumber(c.queryInt("limit", 6), 1, 24) - c.json(200, {ok: true, projects: relatedProjectEntries(entries, c.param("id"), (c.query("exclude") ?? "").split(","), limit)}) + string id = c.param("id") + c.json(200, {ok: true, projects: relatedProjectEntries(entries, id, (c.query("exclude") ?? "").split(","), coPlayedIds(signals, id, 200), limit)}) ) def handleFeaturedProject(*serve.Context c) ( - array entries = projectsVisibleToEveryone(sharedIndexEntries()) - entries.sortBy("weekViews", "descending") + object[] entries = trendingOrder(projectsVisibleToEveryone(sharedIndexEntries()), currentPlaySignals()) array pool = [] for i entries.len ( object entry = entries[i] diff --git a/src/projects/related-projects.osl b/src/projects/related-projects.osl index 1416b38..f632db1 100644 --- a/src/projects/related-projects.osl +++ b/src/projects/related-projects.osl @@ -1,62 +1,90 @@ export {} -def relatedProjectEntries(array entries, string id, array excluded, number limit) array ( - object seen = {} - seen[id] = true +class RelatedPick ( + object[] related = [] + string[boolean] seen = {} + number limit = 6 +) + +def pickRelated(RelatedPick pick, object entry) boolean ( + string entryId = entry.id.toStr() + if pick.related.len >= pick.limit or pick.seen.contains(entryId) ( + return false + ) + pick.seen[entryId] = true + pick.related.append(entry) + return true +) + +def sharesTag(object entry, string[] tags) boolean ( + if !entry.contains("tags") or entry.tags == null ( + return false + ) + array entryTags = entry.tags.assert(array) + for j tags.len ( + if entryTags.contains(tags[j]) ( + return true + ) + ) + return false +) + +def relatedProjectEntries(object[] entries, string id, string[] excluded, string[] coPlayed, number limit) object[] ( + RelatedPick pick = RelatedPick() + pick.limit = limit + pick.seen[id] = true for i excluded.len ( - string excludedId = excluded[i].toStr().strip() + string excludedId = excluded[i].strip() if excludedId != "" ( - seen[excludedId] = true + pick.seen[excludedId] = true ) ) string ownerKey = "" - array tags = [] + string[] tags = [] + string[object] byId = {} for i entries.len ( object entry = entries[i] + byId[entry.id.toStr()] = entry if entry.id.toStr() == id ( ownerKey = entry.owner.toStr().toLower() - if entry.contains("tags") and entry.tags != null ( - tags = entry.tags.assert(array) - ) + tags = sharedTagList(entry) ) ) number ownerLimit = (limit / 2).toInt() if ownerLimit < 1 ( ownerLimit = 1 ) - array related = [] + number coPlayLimit = ownerLimit + for i coPlayed.len ( + if pick.related.len < coPlayLimit and byId.contains(coPlayed[i]) ( + void pickRelated(pick, byId[coPlayed[i]].assert(object)) + ) + ) + number ownerCount = 0 for i entries.len ( object entry = entries[i] - string entryId = entry.id.toStr() - if related.len < ownerLimit and !seen.contains(entryId) and ownerKey != "" and entry.owner.toStr().toLower() == ownerKey ( - seen[entryId] = true - related.append(entry) + if ownerCount < ownerLimit and ownerKey != "" and entry.owner.toStr().toLower() == ownerKey and pickRelated(pick, entry) ( + ownerCount += 1 ) ) for i entries.len ( - object entry = entries[i] - string entryId = entry.id.toStr() - if related.len < limit and !seen.contains(entryId) and entry.contains("tags") and entry.tags != null ( - array entryTags = entry.tags.assert(array) - boolean shares = false - for j tags.len ( - if entryTags.contains(tags[j]) ( - shares = true - ) - ) - if shares ( - seen[entryId] = true - related.append(entry) - ) + if sharesTag(entries[i], tags) ( + void pickRelated(pick, entries[i]) ) ) for i entries.len ( - object entry = entries[i] - string entryId = entry.id.toStr() - if related.len < limit and !seen.contains(entryId) ( - seen[entryId] = true - related.append(entry) + void pickRelated(pick, entries[i]) + ) + return pick.related +) + +def sharedTagList(object entry) string[] ( + string[] tags = [] + if entry.contains("tags") and entry.tags != null ( + array raw = entry.tags.assert(array) + for i raw.len ( + tags.append(raw[i].toStr()) ) ) - return related + return tags ) diff --git a/src/projects/related-projects.test.osl b/src/projects/related-projects.test.osl index 2900fc9..683683c 100644 --- a/src/projects/related-projects.test.osl +++ b/src/projects/related-projects.test.osl @@ -1,7 +1,7 @@ import "std:testing" import "./related-projects.osl" -array entries = [ +object[] entries = [ {id: "a", owner: "Mist", tags: ["games"]}, {id: "b", owner: "mist", tags: ["art"]}, {id: "c", owner: "mist", tags: ["art"]}, @@ -10,7 +10,7 @@ array entries = [ {id: "f", owner: "third", tags: []} ] -def ids(array projects) string ( +def ids(object[] projects) string ( string res = "" for i projects.len ( object project = projects[i] @@ -19,7 +19,11 @@ def ids(array projects) string ( return res ) -testing.assert(ids(relatedProjectEntries(entries, "a", [], 4)) == "bcde", "same owner fills half, then shared tags, then trending") -testing.assert(ids(relatedProjectEntries(entries, "a", ["b", "d"], 3)) == "cef", "excluded projects are skipped") -testing.assert(ids(relatedProjectEntries(entries, "missing", [], 2)) == "ab", "unknown projects fall back to trending order") -testing.assert(ids(relatedProjectEntries(entries, "a", [], 24)) == "bcdef", "the current project never appears") +testing.assert(ids(relatedProjectEntries(entries, "a", [], [], 4)) == "bcde", "same owner fills half, then shared tags, then trending") +testing.assert(ids(relatedProjectEntries(entries, "a", ["b", "d"], [], 3)) == "cef", "excluded projects are skipped") +testing.assert(ids(relatedProjectEntries(entries, "missing", [], [], 2)) == "ab", "unknown projects fall back to trending order") +testing.assert(ids(relatedProjectEntries(entries, "a", [], [], 24)) == "bcdef", "the current project never appears") +testing.assert(ids(relatedProjectEntries(entries, "a", [], ["f", "e"], 4)) == "febc", "projects the same players stayed in come first, up to half the list") +testing.assert(ids(relatedProjectEntries(entries, "a", [], ["f", "e", "d"], 4)) == "febc", "co-played projects never take more than half the list") +testing.assert(ids(relatedProjectEntries(entries, "a", ["f"], ["f", "gone", "a"], 3)) == "bdc", "co-played projects skip excluded, hidden and current ones") +testing.assert(ids(relatedProjectEntries(entries, "a", [], ["b", "c"], 4)) == "bcde", "co-played projects by the same owner still leave room for others") diff --git a/src/storage/storage.osl b/src/storage/storage.osl index 6b2abc9..c48db16 100644 --- a/src/storage/storage.osl +++ b/src/storage/storage.osl @@ -183,6 +183,10 @@ def indexEntryFromProject(object project) object ( if project.contains("groupTag") and project.groupTag != null ( groupTag = normalizeGroupTag(project.groupTag.toStr()) ) + number lastUploadAt = 0 + if project.contains("lastUploadAt") and project.lastUploadAt != null ( + lastUploadAt = project.lastUploadAt.toNum() + ) string vanitySlug = "" if project.contains("vanitySlug") and project.vanitySlug != null ( vanitySlug = project.vanitySlug.toStr() @@ -208,6 +212,7 @@ def indexEntryFromProject(object project) object ( created: project.created, edited: project.edited, sharedAt: project.sharedAt, + lastUploadAt: lastUploadAt, sizeBytes: projectSizeBytes(project), historyBytes: projectHistoryBytes(project), weekViews: weekViewsOf(project),