diff --git a/opal.json b/opal.json index be3259a..bbbb63e 100644 --- a/opal.json +++ b/opal.json @@ -1,6 +1,6 @@ { "name": "mistwarp-api", - "version": "1.0.1", + "version": "1.1.0", "main": "main.osl", "scripts": { "dev": ["osl", "run", "main.osl"], diff --git a/src/admin/admin.osl b/src/admin/admin.osl index 8819839..3e1574c 100644 --- a/src/admin/admin.osl +++ b/src/admin/admin.osl @@ -7,6 +7,7 @@ export { loadBans, projectHiddenFrom, projectOwnerBanned, + playerUnbanned, projectsVisibleTo, projectsVisibleToEveryone, saveAdmins, @@ -103,6 +104,10 @@ def projectsVisibleTo(array entries,*serve.Context c) array ( return projectEntriesListed(entries, bannedOwners(), shadowBannedSet(), normalizeUsername(contextString(c, "username")), contextString(c, "userId"), contextBool(c, "isAdmin")) ) +def playerUnbanned(string username, string userId) boolean ( + return !ownerInBannedSet(bannedOwners(), username, userId) and !ownerInBannedSet(shadowBannedSet(), username, userId) +) + def projectsVisibleToEveryone(array entries) array ( return projectEntriesListed(entries, bannedOwners(), shadowBannedSet(), "", "", false) ) diff --git a/src/admin/funnel-events-policy.osl b/src/admin/funnel-events-policy.osl index 36eeba8..d18bfb5 100644 --- a/src/admin/funnel-events-policy.osl +++ b/src/admin/funnel-events-policy.osl @@ -4,6 +4,8 @@ object funnelEventVariants = { editor_open: [], home_view: [], project_view: ["signed_in", "signed_out"], + visit_start: ["new", "returning"], + visit_depth: ["2", "3", "5", "10"], project_play_start: [], start_creating: ["home"], signin_prompt_shown: ["remix", "save", "library", "react", "comment", "follow"], diff --git a/src/admin/funnel-events-policy.test.osl b/src/admin/funnel-events-policy.test.osl index d91e28c..8df8581 100644 --- a/src/admin/funnel-events-policy.test.osl +++ b/src/admin/funnel-events-policy.test.osl @@ -13,6 +13,9 @@ testing.assert(funnelEventKey("made_up", "") == "", "unknown events are rejected testing.assert(funnelEventKey("", "remix") == "", "empty names are rejected") testing.assert(funnelEventKey("share_opened", "update") == "share_opened:update", "share opens count by action") testing.assert(funnelEventKey("starter_opened", "clicker") == "starter_opened:clicker", "starter projects count by id") +testing.assert(funnelEventKey("visit_start", "returning") == "visit_start:returning", "visits count new and returning devices") +testing.assert(funnelEventKey("visit_depth", "5") == "visit_depth:5", "visits count how many projects they reached") +testing.assert(funnelEventKey("visit_depth", "4") == "visit_depth", "unlisted depths fall back to the event name") testing.assert(isFunnelOrigin("https://mistwarp.org"), "the community site can count") testing.assert(isFunnelOrigin("https://editor.mistwarp.org"), "the editor origin can count") testing.assert(!isFunnelOrigin("http://mistwarp.org"), "plain http is rejected") diff --git a/src/git/pulls.osl b/src/git/pulls.osl index 614c3d3..6a64433 100644 --- a/src/git/pulls.osl +++ b/src/git/pulls.osl @@ -897,6 +897,7 @@ def fastForwardPullLocked(*serve.Context c, object pull, object target, object s target.assets = source.assets target.sizeBytes = projectSizeBytes(source) target.edited = timestamp + target.lastUploadAt = timestamp if source.contains("jsonBytes") ( target.jsonBytes = source.jsonBytes ) diff --git a/src/projects/play-signals.osl b/src/projects/play-signals.osl new file mode 100644 index 0000000..0b749a2 --- /dev/null +++ b/src/projects/play-signals.osl @@ -0,0 +1,143 @@ +export {} + +number heldPlayMs = 30000 +number playWindowMs = 7776000000 +number minHeldPlayers = 5 +number minCoPlayers = 3 + +class PlaySignals ( + string[number] players = {} + string[number] held = {} + string[string[]] heldBy = {} + string[string[]] heldProjects = {} +) + +struct RankedId ( + string id = "" + number score = 0 +) + +struct RankedEntry ( + number index = 0 + number rank = 0 + number views = 0 +) + +def playerKeyOf(string userId, string username) string ( + if userId.strip() != "" ( + return userId.strip() + ) + return username.toLower().strip() +) + +def addHeldPlay(PlaySignals signals, string projectId, string player) ( + signals.held[projectId] = (signals.held[projectId] ?? 0) + 1 + string[] byProject = signals.heldBy[projectId] ?? [] + byProject.append(player) + signals.heldBy[projectId] = byProject + string[] byPlayer = signals.heldProjects[player] ?? [] + byPlayer.append(projectId) + signals.heldProjects[player] = byPlayer +) + +def playSignalsFrom(object[] rows, string[string] ownerKeys, number now) PlaySignals ( + PlaySignals signals = PlaySignals() + for i rows.len ( + object row = rows[i] + string projectId = (row.projectId ?? "").toStr() + string username = (row.username ?? "").toStr() + string player = playerKeyOf((row.userId ?? "").toStr(), username) + string owner = ownerKeys[projectId] ?? "" + boolean ownPlay = owner != "" and username.toLower().strip() == owner + boolean recent = now - (row.updated ?? 0).toNum() <= playWindowMs + if projectId != "" and player != "" and !ownPlay and recent ( + signals.players[projectId] = (signals.players[projectId] ?? 0) + 1 + if (row.duration ?? 0).toNum() >= heldPlayMs ( + addHeldPlay(signals, projectId, player) + ) + ) + ) + return signals +) + +def holdWeight(PlaySignals signals, string projectId) number ( + number playerCount = signals.players[projectId] ?? 0 + number heldCount = signals.held[projectId] ?? 0 + return (heldCount + 2) / (playerCount + 4) +) + +def compareRankedEntries(RankedEntry a, RankedEntry b) int ( + if a.rank != b.rank ( + return if a.rank > b.rank then -1 else 1 + ) + if a.views != b.views ( + return if a.views > b.views then -1 else 1 + ) + return (a.index - b.index).toInt() +) + +def trendingOrder(object[] entries, PlaySignals signals) object[] ( + RankedEntry[] ranked = [] + for i entries.len ( + object entry = entries[i] + number views = (entry.weekViews ?? 0).toNum() + ranked.append(RankedEntry(i, views * holdWeight(signals, entry.id.toStr()), views)) + ) + RankedEntry[] sorted = ranked.sort(compareRankedEntries) + object[] ordered = [] + for i sorted.len ( + ordered.append(entries[sorted[i].index]) + ) + return ordered +) + +def compareRankedIds(RankedId a, RankedId b) int ( + if a.score != b.score ( + return if a.score > b.score then -1 else 1 + ) + if a.id == b.id ( + return 0 + ) + return if a.id < b.id then -1 else 1 +) + +def coPlayScores(PlaySignals signals, string projectId) object ( + string[] sharedPlayers = signals.heldBy[projectId] ?? [] + string[number] scores = {} + string[number] counts = {} + for i sharedPlayers.len ( + string[] playedNext = signals.heldProjects[sharedPlayers[i]] ?? [] + number share = 1 / (playedNext.len.toNum().sqrt() ?? 1) + for j playedNext.len ( + string otherId = playedNext[j] + if otherId != projectId ( + scores[otherId] = (scores[otherId] ?? 0) + share + counts[otherId] = (counts[otherId] ?? 0) + 1 + ) + ) + ) + return {scores: scores, counts: counts} +) + +def coPlayedIds(PlaySignals signals, string projectId, number limit) string[] ( + string[] ids = [] + if (signals.held[projectId] ?? 0) < minHeldPlayers ( + return ids + ) + object found = coPlayScores(signals, projectId) + string[number] scores = found.scores.assert(string[number]) + string[number] counts = found.counts.assert(string[number]) + RankedId[] ranked = [] + for otherId of scores ( + if (counts[otherId] ?? 0) >= minCoPlayers ( + ranked.append(RankedId(otherId, scores[otherId] ?? 0)) + ) + ) + RankedId[] sorted = ranked.sort(compareRankedIds) + for i sorted.len ( + if ids.len < limit ( + ids.append(sorted[i].id) + ) + ) + return ids +) diff --git a/src/projects/play-signals.test.osl b/src/projects/play-signals.test.osl new file mode 100644 index 0000000..56f2610 --- /dev/null +++ b/src/projects/play-signals.test.osl @@ -0,0 +1,66 @@ +import "std:testing" +import "./play-signals.osl" + +number now = 8000000000000 + +def play(string projectId, string player, number duration) object ( + return {_id: projectId ++ ":" ++ player, projectId: projectId, userId: player, username: player, duration: duration, updated: now} +) + +def ids(string[] items) string ( + string res = "" + for i items.len ( + res ++= items[i] ++ " " + ) + return res.strip() +) + +def entryIds(object[] items) string ( + string res = "" + for i items.len ( + object item = items[i] + res ++= item.id.toStr() + ) + return res +) + +object[] rows = [] +for i 5 ( + string player = "fan" ++ i.toStr() + rows.append(play("bait", player, 60000)) + if i <= 3 ( + rows.append(play("pair", player, 60000)) + ) + if i <= 2 ( + rows.append(play("thin", player, 60000)) + ) +) +rows.append(play("bait", "maker", 900000)) +rows.append(play("solo", "maker", 900000)) +rows.append(play("bait", "skimmer", 2000)) +rows.append({_id: "bait:old", projectId: "bait", userId: "old", username: "old", duration: 60000, updated: now - 9000000000}) +rows.append({_id: "pair:legacy", projectId: "pair", userId: "", username: " Legacy ", duration: 60000, updated: now}) +string[string] owners = {bait: "maker", solo: "maker", pair: "someone", thin: "someone"} +PlaySignals signals = playSignalsFrom(rows, owners, now) + +testing.assert(holdWeight(signals, "bait") == 7 / 10, "held plays raise the weight, the owner's and old plays do not count") +testing.assert(holdWeight(signals, "unplayed") == 0.5, "projects nobody has played sit in the middle") +testing.assert(ids(coPlayedIds(signals, "bait", 5)) == "pair", "a project needs three shared players to count as co-played") +testing.assert(ids(coPlayedIds(signals, "thin", 5)) == "", "projects with few held players get no co-plays") +testing.assert(ids(coPlayedIds(signals, "solo", 5)) == "", "the owner's own plays never link projects") +testing.assert((signals.players["pair"] ?? 0) == 4, "legacy rows without a user id count once under their username") + +object[] tied = [] +for i 30 ( + tied.append({id: "t" ++ (100 + i).toStr(), weekViews: 5}) +) +testing.assert(entryIds(trendingOrder(tied, PlaySignals())).startsWith("t101t102t103t104t105t106t107t108t109t110t111t112t113t114t115"), "equal ranks keep their incoming order on long lists") + +object[] entries = [ + {id: "pair", weekViews: 10}, + {id: "bait", weekViews: 9}, + {id: "x", weekViews: 9}, + {id: "z", weekViews: 0} +] +testing.assert(entryIds(trendingOrder(entries, signals)) == "pairbaitxz", "views are weighted by how long plays last") +testing.assert(entryIds(trendingOrder(entries, PlaySignals())) == "pairbaitxz", "without playtime the order follows views") diff --git a/src/projects/project-discovery.osl b/src/projects/project-discovery.osl index a91284a..8a0566e 100644 --- a/src/projects/project-discovery.osl +++ b/src/projects/project-discovery.osl @@ -5,6 +5,9 @@ export { // osl-disable-next-line OSL-UNTYPED-GLOBAL auto sharedProjectsCache = cache.new({capacity: 2, ttl: 60000}) +// osl-disable-next-line OSL-UNTYPED-GLOBAL +auto playSignalsCache = cache.new({capacity: 2, ttl: 300000}) + def sharedIndexEntries() array ( if sharedProjectsCache.has("projects") ( return sharedProjectsCache.get("projects").assert(array) @@ -21,6 +24,51 @@ def sharedIndexEntries() array ( return entries ) +def sharedPlayRows(array rows) object[] ( + string[boolean] allowed = {} + object[] kept = [] + for i rows.len ( + object row = rows[i].assert(object) + string username = (row.username ?? "").toStr() + string key = normalizeUsername(username) + if key != "" and !allowed.contains(key) ( + allowed[key] = playerUnbanned(username, (row.userId ?? "").toStr()) and effectiveShowRecentActivity(loadProfile(username)) + ) + if key != "" and (allowed[key] ?? false) ( + kept.append(row) + ) + ) + return kept +) + +def buildPlaySignals() PlaySignals ( + string[string] ownerKeys = {} + array entries = sharedIndexEntries() + for i entries.len ( + object entry = entries[i] + ownerKeys[entry.id.toStr()] = (entry.ownerKey ?? entry.owner ?? "").toStr().toLower().strip() + ) + array rows = playtimeDocs.all() catch (err) ( + log "WARNING: could not read playtime for discovery: " ++ err.toStr() + return PlaySignals() + ) + return playSignalsFrom(sharedPlayRows(rows), ownerKeys, timestamp) +) + +def currentPlaySignals() PlaySignals ( + if playSignalsCache.has("signals") ( + return playSignalsCache.get("signals").assert(PlaySignals) + ) + sync.lock("play-signals") + defer sync.unlock("play-signals") + if playSignalsCache.has("signals") ( + return playSignalsCache.get("signals").assert(PlaySignals) + ) + PlaySignals signals = buildPlaySignals() + playSignalsCache.set("signals", signals) + return signals +) + def handleExplore(*serve.Context c) ( array entries = projectsVisibleTo(sharedIndexEntries(), c) @@ -87,7 +135,7 @@ def handleExplore(*serve.Context c) ( ) else if sort == "undiscovered" ( entries.sortBy("views", "ascending") ) else if sort == "trending" ( - entries.sortBy("weekViews", "descending") + entries = trendingOrder(entries, currentPlaySignals()) ) else ( entries.sortBy("sharedAt", "descending") ) @@ -109,15 +157,15 @@ def handleRandomProject(*serve.Context c) ( ) def handleRelatedProjects(*serve.Context c) ( - array entries = projectsVisibleToEveryone(sharedIndexEntries()) - entries.sortBy("weekViews", "descending") + PlaySignals signals = currentPlaySignals() + object[] entries = trendingOrder(projectsVisibleToEveryone(sharedIndexEntries()), signals) number limit = clampNumber(c.queryInt("limit", 6), 1, 24) - c.json(200, {ok: true, projects: relatedProjectEntries(entries, c.param("id"), (c.query("exclude") ?? "").split(","), limit)}) + string id = c.param("id") + c.json(200, {ok: true, projects: relatedProjectEntries(entries, id, (c.query("exclude") ?? "").split(","), coPlayedIds(signals, id, 200), limit)}) ) def handleFeaturedProject(*serve.Context c) ( - array entries = projectsVisibleToEveryone(sharedIndexEntries()) - entries.sortBy("weekViews", "descending") + object[] entries = trendingOrder(projectsVisibleToEveryone(sharedIndexEntries()), currentPlaySignals()) array pool = [] for i entries.len ( object entry = entries[i] diff --git a/src/projects/related-projects.osl b/src/projects/related-projects.osl index 1416b38..f632db1 100644 --- a/src/projects/related-projects.osl +++ b/src/projects/related-projects.osl @@ -1,62 +1,90 @@ export {} -def relatedProjectEntries(array entries, string id, array excluded, number limit) array ( - object seen = {} - seen[id] = true +class RelatedPick ( + object[] related = [] + string[boolean] seen = {} + number limit = 6 +) + +def pickRelated(RelatedPick pick, object entry) boolean ( + string entryId = entry.id.toStr() + if pick.related.len >= pick.limit or pick.seen.contains(entryId) ( + return false + ) + pick.seen[entryId] = true + pick.related.append(entry) + return true +) + +def sharesTag(object entry, string[] tags) boolean ( + if !entry.contains("tags") or entry.tags == null ( + return false + ) + array entryTags = entry.tags.assert(array) + for j tags.len ( + if entryTags.contains(tags[j]) ( + return true + ) + ) + return false +) + +def relatedProjectEntries(object[] entries, string id, string[] excluded, string[] coPlayed, number limit) object[] ( + RelatedPick pick = RelatedPick() + pick.limit = limit + pick.seen[id] = true for i excluded.len ( - string excludedId = excluded[i].toStr().strip() + string excludedId = excluded[i].strip() if excludedId != "" ( - seen[excludedId] = true + pick.seen[excludedId] = true ) ) string ownerKey = "" - array tags = [] + string[] tags = [] + string[object] byId = {} for i entries.len ( object entry = entries[i] + byId[entry.id.toStr()] = entry if entry.id.toStr() == id ( ownerKey = entry.owner.toStr().toLower() - if entry.contains("tags") and entry.tags != null ( - tags = entry.tags.assert(array) - ) + tags = sharedTagList(entry) ) ) number ownerLimit = (limit / 2).toInt() if ownerLimit < 1 ( ownerLimit = 1 ) - array related = [] + number coPlayLimit = ownerLimit + for i coPlayed.len ( + if pick.related.len < coPlayLimit and byId.contains(coPlayed[i]) ( + void pickRelated(pick, byId[coPlayed[i]].assert(object)) + ) + ) + number ownerCount = 0 for i entries.len ( object entry = entries[i] - string entryId = entry.id.toStr() - if related.len < ownerLimit and !seen.contains(entryId) and ownerKey != "" and entry.owner.toStr().toLower() == ownerKey ( - seen[entryId] = true - related.append(entry) + if ownerCount < ownerLimit and ownerKey != "" and entry.owner.toStr().toLower() == ownerKey and pickRelated(pick, entry) ( + ownerCount += 1 ) ) for i entries.len ( - object entry = entries[i] - string entryId = entry.id.toStr() - if related.len < limit and !seen.contains(entryId) and entry.contains("tags") and entry.tags != null ( - array entryTags = entry.tags.assert(array) - boolean shares = false - for j tags.len ( - if entryTags.contains(tags[j]) ( - shares = true - ) - ) - if shares ( - seen[entryId] = true - related.append(entry) - ) + if sharesTag(entries[i], tags) ( + void pickRelated(pick, entries[i]) ) ) for i entries.len ( - object entry = entries[i] - string entryId = entry.id.toStr() - if related.len < limit and !seen.contains(entryId) ( - seen[entryId] = true - related.append(entry) + void pickRelated(pick, entries[i]) + ) + return pick.related +) + +def sharedTagList(object entry) string[] ( + string[] tags = [] + if entry.contains("tags") and entry.tags != null ( + array raw = entry.tags.assert(array) + for i raw.len ( + tags.append(raw[i].toStr()) ) ) - return related + return tags ) diff --git a/src/projects/related-projects.test.osl b/src/projects/related-projects.test.osl index 2900fc9..683683c 100644 --- a/src/projects/related-projects.test.osl +++ b/src/projects/related-projects.test.osl @@ -1,7 +1,7 @@ import "std:testing" import "./related-projects.osl" -array entries = [ +object[] entries = [ {id: "a", owner: "Mist", tags: ["games"]}, {id: "b", owner: "mist", tags: ["art"]}, {id: "c", owner: "mist", tags: ["art"]}, @@ -10,7 +10,7 @@ array entries = [ {id: "f", owner: "third", tags: []} ] -def ids(array projects) string ( +def ids(object[] projects) string ( string res = "" for i projects.len ( object project = projects[i] @@ -19,7 +19,11 @@ def ids(array projects) string ( return res ) -testing.assert(ids(relatedProjectEntries(entries, "a", [], 4)) == "bcde", "same owner fills half, then shared tags, then trending") -testing.assert(ids(relatedProjectEntries(entries, "a", ["b", "d"], 3)) == "cef", "excluded projects are skipped") -testing.assert(ids(relatedProjectEntries(entries, "missing", [], 2)) == "ab", "unknown projects fall back to trending order") -testing.assert(ids(relatedProjectEntries(entries, "a", [], 24)) == "bcdef", "the current project never appears") +testing.assert(ids(relatedProjectEntries(entries, "a", [], [], 4)) == "bcde", "same owner fills half, then shared tags, then trending") +testing.assert(ids(relatedProjectEntries(entries, "a", ["b", "d"], [], 3)) == "cef", "excluded projects are skipped") +testing.assert(ids(relatedProjectEntries(entries, "missing", [], [], 2)) == "ab", "unknown projects fall back to trending order") +testing.assert(ids(relatedProjectEntries(entries, "a", [], [], 24)) == "bcdef", "the current project never appears") +testing.assert(ids(relatedProjectEntries(entries, "a", [], ["f", "e"], 4)) == "febc", "projects the same players stayed in come first, up to half the list") +testing.assert(ids(relatedProjectEntries(entries, "a", [], ["f", "e", "d"], 4)) == "febc", "co-played projects never take more than half the list") +testing.assert(ids(relatedProjectEntries(entries, "a", ["f"], ["f", "gone", "a"], 3)) == "bdc", "co-played projects skip excluded, hidden and current ones") +testing.assert(ids(relatedProjectEntries(entries, "a", [], ["b", "c"], 4)) == "bcde", "co-played projects by the same owner still leave room for others") diff --git a/src/storage/storage.osl b/src/storage/storage.osl index 6b2abc9..c48db16 100644 --- a/src/storage/storage.osl +++ b/src/storage/storage.osl @@ -183,6 +183,10 @@ def indexEntryFromProject(object project) object ( if project.contains("groupTag") and project.groupTag != null ( groupTag = normalizeGroupTag(project.groupTag.toStr()) ) + number lastUploadAt = 0 + if project.contains("lastUploadAt") and project.lastUploadAt != null ( + lastUploadAt = project.lastUploadAt.toNum() + ) string vanitySlug = "" if project.contains("vanitySlug") and project.vanitySlug != null ( vanitySlug = project.vanitySlug.toStr() @@ -208,6 +212,7 @@ def indexEntryFromProject(object project) object ( created: project.created, edited: project.edited, sharedAt: project.sharedAt, + lastUploadAt: lastUploadAt, sizeBytes: projectSizeBytes(project), historyBytes: projectHistoryBytes(project), weekViews: weekViewsOf(project),