Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion opal.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "mistwarp-api",
"version": "1.0.1",
"version": "1.1.0",
"main": "main.osl",
"scripts": {
"dev": ["osl", "run", "main.osl"],
Expand Down
5 changes: 5 additions & 0 deletions src/admin/admin.osl
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ export {
loadBans,
projectHiddenFrom,
projectOwnerBanned,
playerUnbanned,
projectsVisibleTo,
projectsVisibleToEveryone,
saveAdmins,
Expand Down Expand Up @@ -103,6 +104,10 @@ def projectsVisibleTo(array entries,*serve.Context c) array (
return projectEntriesListed(entries, bannedOwners(), shadowBannedSet(), normalizeUsername(contextString(c, "username")), contextString(c, "userId"), contextBool(c, "isAdmin"))
)

def playerUnbanned(string username, string userId) boolean (
return !ownerInBannedSet(bannedOwners(), username, userId) and !ownerInBannedSet(shadowBannedSet(), username, userId)
)

def projectsVisibleToEveryone(array entries) array (
return projectEntriesListed(entries, bannedOwners(), shadowBannedSet(), "", "", false)
)
Expand Down
2 changes: 2 additions & 0 deletions src/admin/funnel-events-policy.osl
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ object funnelEventVariants = {
editor_open: [],
home_view: [],
project_view: ["signed_in", "signed_out"],
visit_start: ["new", "returning"],
visit_depth: ["2", "3", "5", "10"],
project_play_start: [],
start_creating: ["home"],
signin_prompt_shown: ["remix", "save", "library", "react", "comment", "follow"],
Expand Down
3 changes: 3 additions & 0 deletions src/admin/funnel-events-policy.test.osl
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ testing.assert(funnelEventKey("made_up", "") == "", "unknown events are rejected
testing.assert(funnelEventKey("", "remix") == "", "empty names are rejected")
testing.assert(funnelEventKey("share_opened", "update") == "share_opened:update", "share opens count by action")
testing.assert(funnelEventKey("starter_opened", "clicker") == "starter_opened:clicker", "starter projects count by id")
testing.assert(funnelEventKey("visit_start", "returning") == "visit_start:returning", "visits count new and returning devices")
testing.assert(funnelEventKey("visit_depth", "5") == "visit_depth:5", "visits count how many projects they reached")
testing.assert(funnelEventKey("visit_depth", "4") == "visit_depth", "unlisted depths fall back to the event name")
testing.assert(isFunnelOrigin("https://mistwarp.org"), "the community site can count")
testing.assert(isFunnelOrigin("https://editor.mistwarp.org"), "the editor origin can count")
testing.assert(!isFunnelOrigin("http://mistwarp.org"), "plain http is rejected")
Expand Down
1 change: 1 addition & 0 deletions src/git/pulls.osl
Original file line number Diff line number Diff line change
Expand Up @@ -897,6 +897,7 @@ def fastForwardPullLocked(*serve.Context c, object pull, object target, object s
target.assets = source.assets
target.sizeBytes = projectSizeBytes(source)
target.edited = timestamp
target.lastUploadAt = timestamp
if source.contains("jsonBytes") (
target.jsonBytes = source.jsonBytes
)
Expand Down
143 changes: 143 additions & 0 deletions src/projects/play-signals.osl
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
export {}

number heldPlayMs = 30000
number playWindowMs = 7776000000
number minHeldPlayers = 5
number minCoPlayers = 3

class PlaySignals (
string[number] players = {}
string[number] held = {}
string[string[]] heldBy = {}
string[string[]] heldProjects = {}
)

struct RankedId (
string id = ""
number score = 0
)

struct RankedEntry (
number index = 0
number rank = 0
number views = 0
)

def playerKeyOf(string userId, string username) string (
if userId.strip() != "" (
return userId.strip()
)
return username.toLower().strip()
)

def addHeldPlay(PlaySignals signals, string projectId, string player) (
signals.held[projectId] = (signals.held[projectId] ?? 0) + 1
string[] byProject = signals.heldBy[projectId] ?? []
byProject.append(player)
signals.heldBy[projectId] = byProject
string[] byPlayer = signals.heldProjects[player] ?? []
byPlayer.append(projectId)
signals.heldProjects[player] = byPlayer
)

def playSignalsFrom(object[] rows, string[string] ownerKeys, number now) PlaySignals (
PlaySignals signals = PlaySignals()
for i rows.len (
object row = rows[i]
string projectId = (row.projectId ?? "").toStr()
string username = (row.username ?? "").toStr()
string player = playerKeyOf((row.userId ?? "").toStr(), username)
string owner = ownerKeys[projectId] ?? ""
boolean ownPlay = owner != "" and username.toLower().strip() == owner
boolean recent = now - (row.updated ?? 0).toNum() <= playWindowMs
if projectId != "" and player != "" and !ownPlay and recent (
signals.players[projectId] = (signals.players[projectId] ?? 0) + 1
if (row.duration ?? 0).toNum() >= heldPlayMs (
addHeldPlay(signals, projectId, player)
)
)
)
return signals
)

def holdWeight(PlaySignals signals, string projectId) number (
number playerCount = signals.players[projectId] ?? 0
number heldCount = signals.held[projectId] ?? 0
return (heldCount + 2) / (playerCount + 4)
)

def compareRankedEntries(RankedEntry a, RankedEntry b) int (
if a.rank != b.rank (
return if a.rank > b.rank then -1 else 1
)
if a.views != b.views (
return if a.views > b.views then -1 else 1
)
return (a.index - b.index).toInt()
)

def trendingOrder(object[] entries, PlaySignals signals) object[] (
RankedEntry[] ranked = []
for i entries.len (
object entry = entries[i]
number views = (entry.weekViews ?? 0).toNum()
ranked.append(RankedEntry(i, views * holdWeight(signals, entry.id.toStr()), views))
)
RankedEntry[] sorted = ranked.sort(compareRankedEntries)
object[] ordered = []
for i sorted.len (
ordered.append(entries[sorted[i].index])
)
return ordered
)

def compareRankedIds(RankedId a, RankedId b) int (
if a.score != b.score (
return if a.score > b.score then -1 else 1
)
if a.id == b.id (
return 0
)
return if a.id < b.id then -1 else 1
)

def coPlayScores(PlaySignals signals, string projectId) object (
string[] sharedPlayers = signals.heldBy[projectId] ?? []
string[number] scores = {}
string[number] counts = {}
for i sharedPlayers.len (
string[] playedNext = signals.heldProjects[sharedPlayers[i]] ?? []
number share = 1 / (playedNext.len.toNum().sqrt() ?? 1)
for j playedNext.len (
string otherId = playedNext[j]
if otherId != projectId (
scores[otherId] = (scores[otherId] ?? 0) + share
counts[otherId] = (counts[otherId] ?? 0) + 1
)
)
)
return {scores: scores, counts: counts}
)

def coPlayedIds(PlaySignals signals, string projectId, number limit) string[] (
string[] ids = []
if (signals.held[projectId] ?? 0) < minHeldPlayers (
return ids
)
object found = coPlayScores(signals, projectId)
string[number] scores = found.scores.assert(string[number])
string[number] counts = found.counts.assert(string[number])
RankedId[] ranked = []
for otherId of scores (
if (counts[otherId] ?? 0) >= minCoPlayers (
ranked.append(RankedId(otherId, scores[otherId] ?? 0))
)
)
RankedId[] sorted = ranked.sort(compareRankedIds)
for i sorted.len (
if ids.len < limit (
ids.append(sorted[i].id)
)
)
return ids
)
66 changes: 66 additions & 0 deletions src/projects/play-signals.test.osl
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import "std:testing"
import "./play-signals.osl"

number now = 8000000000000

def play(string projectId, string player, number duration) object (
return {_id: projectId ++ ":" ++ player, projectId: projectId, userId: player, username: player, duration: duration, updated: now}
)

def ids(string[] items) string (
string res = ""
for i items.len (
res ++= items[i] ++ " "
)
return res.strip()
)

def entryIds(object[] items) string (
string res = ""
for i items.len (
object item = items[i]
res ++= item.id.toStr()
)
return res
)

object[] rows = []
for i 5 (
string player = "fan" ++ i.toStr()
rows.append(play("bait", player, 60000))
if i <= 3 (
rows.append(play("pair", player, 60000))
)
if i <= 2 (
rows.append(play("thin", player, 60000))
)
)
rows.append(play("bait", "maker", 900000))
rows.append(play("solo", "maker", 900000))
rows.append(play("bait", "skimmer", 2000))
rows.append({_id: "bait:old", projectId: "bait", userId: "old", username: "old", duration: 60000, updated: now - 9000000000})
rows.append({_id: "pair:legacy", projectId: "pair", userId: "", username: " Legacy ", duration: 60000, updated: now})
string[string] owners = {bait: "maker", solo: "maker", pair: "someone", thin: "someone"}
PlaySignals signals = playSignalsFrom(rows, owners, now)

testing.assert(holdWeight(signals, "bait") == 7 / 10, "held plays raise the weight, the owner's and old plays do not count")
testing.assert(holdWeight(signals, "unplayed") == 0.5, "projects nobody has played sit in the middle")
testing.assert(ids(coPlayedIds(signals, "bait", 5)) == "pair", "a project needs three shared players to count as co-played")
testing.assert(ids(coPlayedIds(signals, "thin", 5)) == "", "projects with few held players get no co-plays")
testing.assert(ids(coPlayedIds(signals, "solo", 5)) == "", "the owner's own plays never link projects")
testing.assert((signals.players["pair"] ?? 0) == 4, "legacy rows without a user id count once under their username")

object[] tied = []
for i 30 (
tied.append({id: "t" ++ (100 + i).toStr(), weekViews: 5})
)
testing.assert(entryIds(trendingOrder(tied, PlaySignals())).startsWith("t101t102t103t104t105t106t107t108t109t110t111t112t113t114t115"), "equal ranks keep their incoming order on long lists")

object[] entries = [
{id: "pair", weekViews: 10},
{id: "bait", weekViews: 9},
{id: "x", weekViews: 9},
{id: "z", weekViews: 0}
]
testing.assert(entryIds(trendingOrder(entries, signals)) == "pairbaitxz", "views are weighted by how long plays last")
testing.assert(entryIds(trendingOrder(entries, PlaySignals())) == "pairbaitxz", "without playtime the order follows views")
60 changes: 54 additions & 6 deletions src/projects/project-discovery.osl
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ export {
// osl-disable-next-line OSL-UNTYPED-GLOBAL
auto sharedProjectsCache = cache.new({capacity: 2, ttl: 60000})

// osl-disable-next-line OSL-UNTYPED-GLOBAL
auto playSignalsCache = cache.new({capacity: 2, ttl: 300000})

def sharedIndexEntries() array (
if sharedProjectsCache.has("projects") (
return sharedProjectsCache.get("projects").assert(array)
Expand All @@ -21,6 +24,51 @@ def sharedIndexEntries() array (
return entries
)

def sharedPlayRows(array rows) object[] (
string[boolean] allowed = {}
object[] kept = []
for i rows.len (
object row = rows[i].assert(object)
string username = (row.username ?? "").toStr()
string key = normalizeUsername(username)
if key != "" and !allowed.contains(key) (
allowed[key] = playerUnbanned(username, (row.userId ?? "").toStr()) and effectiveShowRecentActivity(loadProfile(username))
)
if key != "" and (allowed[key] ?? false) (
kept.append(row)
)
)
return kept
)

def buildPlaySignals() PlaySignals (
string[string] ownerKeys = {}
array entries = sharedIndexEntries()
for i entries.len (
object entry = entries[i]
ownerKeys[entry.id.toStr()] = (entry.ownerKey ?? entry.owner ?? "").toStr().toLower().strip()
)
array rows = playtimeDocs.all() catch (err) (
log "WARNING: could not read playtime for discovery: " ++ err.toStr()
return PlaySignals()
)
return playSignalsFrom(sharedPlayRows(rows), ownerKeys, timestamp)
)

def currentPlaySignals() PlaySignals (
if playSignalsCache.has("signals") (
return playSignalsCache.get("signals").assert(PlaySignals)
)
sync.lock("play-signals")
defer sync.unlock("play-signals")
if playSignalsCache.has("signals") (
return playSignalsCache.get("signals").assert(PlaySignals)
)
PlaySignals signals = buildPlaySignals()
playSignalsCache.set("signals", signals)
return signals
)

def handleExplore(*serve.Context c) (
array entries = projectsVisibleTo(sharedIndexEntries(), c)

Expand Down Expand Up @@ -87,7 +135,7 @@ def handleExplore(*serve.Context c) (
) else if sort == "undiscovered" (
entries.sortBy("views", "ascending")
) else if sort == "trending" (
entries.sortBy("weekViews", "descending")
entries = trendingOrder(entries, currentPlaySignals())
) else (
entries.sortBy("sharedAt", "descending")
)
Expand All @@ -109,15 +157,15 @@ def handleRandomProject(*serve.Context c) (
)

def handleRelatedProjects(*serve.Context c) (
array entries = projectsVisibleToEveryone(sharedIndexEntries())
entries.sortBy("weekViews", "descending")
PlaySignals signals = currentPlaySignals()
object[] entries = trendingOrder(projectsVisibleToEveryone(sharedIndexEntries()), signals)
number limit = clampNumber(c.queryInt("limit", 6), 1, 24)
c.json(200, {ok: true, projects: relatedProjectEntries(entries, c.param("id"), (c.query("exclude") ?? "").split(","), limit)})
string id = c.param("id")
c.json(200, {ok: true, projects: relatedProjectEntries(entries, id, (c.query("exclude") ?? "").split(","), coPlayedIds(signals, id, 200), limit)})
)

def handleFeaturedProject(*serve.Context c) (
array entries = projectsVisibleToEveryone(sharedIndexEntries())
entries.sortBy("weekViews", "descending")
object[] entries = trendingOrder(projectsVisibleToEveryone(sharedIndexEntries()), currentPlaySignals())
array pool = []
for i entries.len (
object entry = entries[i]
Expand Down
Loading
Loading