From c1eeff1a3926ecc122587950e04ed065827566b7 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Tue, 13 Oct 2026 22:07:56 +0100 Subject: [PATCH] Add grouped public development Live Feed to Scientist --- .github/workflows/organism_board.yml | 52 +++- README.md | 4 + REPORTING.md | 51 ++++ scripts/activity_feed.py | 349 +++++++++++++++++++++++++++ scripts/live_feed.js | 109 +++++++++ scripts/organism_board.py | 14 +- tests/live_feed_browser.cjs | 58 +++++ tests/test_activity_feed.py | 182 ++++++++++++++ 8 files changed, 805 insertions(+), 14 deletions(-) create mode 100644 scripts/activity_feed.py create mode 100644 scripts/live_feed.js create mode 100644 tests/live_feed_browser.cjs create mode 100644 tests/test_activity_feed.py diff --git a/.github/workflows/organism_board.yml b/.github/workflows/organism_board.yml index 8fd3dd9..e0c6f8c 100644 --- a/.github/workflows/organism_board.yml +++ b/.github/workflows/organism_board.yml @@ -1,19 +1,12 @@ name: Dashboard -# Publishes the PyAutoScientist Dashboard — the umbrella ROUTER over the four organ -# dashboards (scripts/organism_board.py): one row per board with its own live -# headline, a "where to work next" banner keyed off the Heart's verdict, -# published to GitHub Pages + badge.json + the README strip between the -# scientist:begin/end markers. Nothing is recomputed here — each organ's -# board stays the authority; this page only routes. -# -# Six-hourly: the organ boards refresh on their own daily/push cadences, so -# the router follows within hours. board_tests.yml checks fixture renders on PRs; -# this workflow builds and publishes the live page. +# Scientist owns a bounded public development activity index. Owner repositories +# remain authoritative. Five-minute scheduling is best effort; the page exposes +# source coverage and event/collection times separately. on: schedule: - - cron: "17 */6 * * *" + - cron: "*/5 * * * *" workflow_dispatch: permissions: @@ -29,6 +22,7 @@ jobs: board: name: Render + publish the dashboard runs-on: ubuntu-latest + timeout-minutes: 4 environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} @@ -54,10 +48,44 @@ jobs: - run: python -m pip install pyyaml + - name: Restore bounded activity index + run: | + mkdir -p _activity _site + if git ls-remote --exit-code --heads origin refs/heads/scientist-activity-cache > /dev/null; then + git fetch --depth 1 origin refs/heads/scientist-activity-cache + git show FETCH_HEAD:cache.json > _activity/cache.json + git rev-parse FETCH_HEAD > _activity/parent + else + status=$? + if [ "$status" -ne 2 ]; then exit "$status"; fi + fi + + - name: Collect public development activity + env: + GH_TOKEN: ${{ secrets.PAT_PYAUTOLABS || github.token }} + ACTIVITY_BUDGET: ${{ secrets.PAT_PYAUTOLABS && '180' || '60' }} + run: | + python scripts/activity_feed.py --repos PyAutoMind/repos.yaml --cache _activity/cache.json --output _site/activity.json --budget "$ACTIVITY_BUDGET" + + - name: Preserve bounded derived index + if: github.ref == 'refs/heads/main' + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + activity_blob=$(git hash-object -w _activity/cache.json) + activity_tree=$(printf '100644 blob %s\tcache.json\n' "$activity_blob" | git mktree) + if [ -f _activity/parent ]; then + activity_parent=$(cat _activity/parent) + activity_commit=$(git commit-tree "$activity_tree" -p "$activity_parent" -m 'Refresh derived public activity index') + else + activity_commit=$(git commit-tree "$activity_tree" -m 'Begin derived public activity index') + fi + git push origin "$activity_commit:refs/heads/scientist-activity-cache" + - name: Render every surface (public HTTPS only) run: | mkdir -p _site - python scripts/organism_board.py --repos PyAutoMind/repos.yaml --html > _site/index.html + python scripts/organism_board.py --repos PyAutoMind/repos.yaml --activity _site/activity.json --html > _site/index.html python scripts/organism_board.py --repos PyAutoMind/repos.yaml --badge > _site/badge.json python scripts/organism_board.py --repos PyAutoMind/repos.yaml --md > board.md cp board.md _site/dashboard.md diff --git a/README.md b/README.md index ef6c15d..8af68d8 100644 --- a/README.md +++ b/README.md @@ -129,3 +129,7 @@ work and authoritative records remain with each organ. ## Connected projects The dashboard includes expandable project cards linking software, shared dependencies, documentation, tutorials, workspaces, assistants and owner records. See [the project configuration contract](PROJECTS.md) to connect resources; an assistant is optional and source records remain with their owners. + +The Scientist **Live Feed** groups recorded public development activity by task, +with five compact rows and expandable history. Its coverage, refresh behaviour +and limits are documented in [REPORTING.md](REPORTING.md#live-feed-recorded-development-activity). diff --git a/REPORTING.md b/REPORTING.md index 4144920..11c479b 100644 --- a/REPORTING.md +++ b/REPORTING.md @@ -84,3 +84,54 @@ A requested summary is read-only. Follow-up work uses the owning workflow and existing approval rules; the request does not authorize compute, replies, merges or releases. Do not add recurring collection, a second task registry or persistent monitoring to produce a one-off report. + +## Live Feed: recorded development activity + +The canonical Scientist dashboard has a grouped Live Feed between Projects and +Dashboards. It covers public development issues, issue/PR discussion updates, +explicit progress reports, closed tasks and merged PRs in Mind's registered +repositories. Library/workspace tasks route to Mind; organ development routes +to that organ. Scientific run progress, private work, local edits and chat-only +work are outside this first version. Independent instance previews do not yet +include the feed. + +One task occupies one row. A PR is grouped with an issue only when its body has +one unambiguous standalone `Closes #N`, `Fixes #N` or `Resolves #N` directive for +that same repository and the target issue is in the collected evidence. Other +PRs remain separate. An issue being opened is not evidence of work starting; +a closed PR is not a merge, and a merge is not publication or science acceptance. + +Comment bodies are never exported. Recognized headings (`Progress`, `Progress +update`, `In progress`, `Blocked`, `Shipped`, `Workspace PR Created`, `Library PR +Created`) produce short factual stage labels and source links. Other human +comments appear as “Discussion update recorded”; bot comments are excluded. +Editing a comment replaces that event without giving it a new posted time. +The source link is the place to read detail, not an automatic transcript mirror. + +Collection runs in the existing Scientist Pages workflow approximately every +five minutes; scheduling and upstream visibility can delay it. The browser +checks the published same-origin `activity.json` every minute while visible. +There is no GitHub credential or per-visitor GitHub request in the page. A new +snapshot is buffered while a reader has expanded details, focused a control, +scrolled into the feed, or loaded older rows. “Show updates” applies it explicitly. + +The `scientist-activity-cache` branch holds a derived, normalized cache and +per-repository cursors. It is not a task registry and contains no API response +bodies, prompts, tokens or raw comments. Only public repositories independently +verified in the current owner catalog are exported. Failed/inaccessible catalogs +suppress their cached rows. Other source failures retain last-known evidence, +mark it stale and leave the cursor unchanged. Losing the cache starts a disclosed +seven-day backfill; successful collection retains up to 30 days, 1500 events per +repository, 200 published task rows and 20 displayed events per task. + +The collector uses `PAT_PYAUTOLABS` when available (180 requests per cycle), or +the workflow token (60). It rotates repositories when the request budget is +exhausted; a busy source or pagination cap is reported in coverage rather than +claimed complete. The issue and comment endpoints paginate up to three pages +of 100 each. Partial batches retain verified events and reuse already normalized PR +revisions on the next attempt. A source cursor advances only after both +endpoints and the needed PR details have succeeded. Each cycle also has a +two-minute collection time budget. Therefore “every five minutes” is the collection +attempt interval, not a guarantee that every repository is current. Event time, +source check time and page check time remain distinct. Routine feed rebuilds +never create activity events. diff --git a/scripts/activity_feed.py b/scripts/activity_feed.py new file mode 100644 index 0000000..45f3f5b --- /dev/null +++ b/scripts/activity_feed.py @@ -0,0 +1,349 @@ +"""Bounded public development-event index. Owner records remain authoritative. + +Only normalized public fields are persisted: never API bodies, comments or tokens. +A source's cursor advances only after complete pagination. Unavailable/private +sources are removed from the publication, not silently retained as current. +""" +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import html +import json +import os +from pathlib import Path +import re +import time +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +SCHEMA = 1 +MAX_EVENTS = 1500 +MAX_TASKS = 200 +MAX_PAGES = 3 +RETENTION_DAYS = 30 +BOARDS = {x.lower(): x for x in ('Scientist', 'Brain', 'Mind', 'Cortex', 'Memory', 'Eyes', 'Ears', 'Heart', 'Hands', 'Pulse', 'Insight', 'DNA', 'Nerves', 'Broca', 'Gut')} + + +def instant(value): + if not isinstance(value, str): + raise ValueError('missing timestamp') + dt = datetime.fromisoformat(value.replace('Z', '+00:00')) + if dt.tzinfo is None: + raise ValueError('timestamp must include timezone') + return dt.astimezone(timezone.utc) + + +def stamp(dt): + return dt.astimezone(timezone.utc).isoformat(timespec='seconds').replace('+00:00', 'Z') + + +def short(text, limit=110): + return re.sub(r'\s+', ' ', str(text or '')).strip()[:limit] + + +def sources(path): + import yaml + rows = yaml.safe_load(Path(path).read_text())['repos'] + result = {} + for name, row in rows.items(): + repo = row.get('github', '') + if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repo): + continue + organ = str(row.get('organ', 'scientist' if name == 'PyAutoScientist' else 'mind')).lower() + board = organ if organ in BOARDS else 'mind' + result[repo] = {'board': board, 'board_url': f'https://{repo.split("/")[0].lower()}.github.io/PyAuto{BOARDS[board]}/'} + return dict(sorted(result.items())) + + +class CollectionError(Exception): + """Public-safe source failure; never includes response content.""" + + +class GitHub: + def __init__(self, token=None, budget=60): + self.token = token + self.budget = budget + self.requests = 0 + self.deadline = time.monotonic() + 120 + self.expired = False + + def get(self, path): + if time.monotonic() >= self.deadline: + self.expired = True + raise CollectionError('collection time budget reached') + if self.requests >= self.budget: + raise CollectionError('request budget reached') + self.requests += 1 + headers = {'Accept': 'application/vnd.github+json', 'User-Agent': 'PyAutoScientist-live-feed', 'X-GitHub-Api-Version': '2022-11-28'} + if self.token: + headers['Authorization'] = 'Bearer ' + self.token + # Construct endpoints locally; never follow an API-supplied URL with a token. + if not path.startswith(('/repos/', '/users/', '/orgs/')) or '://' in path: + raise CollectionError('invalid API path') + try: + with urlopen(Request('https://api.github.com' + path, headers=headers), timeout=15) as response: + return json.load(response) + except HTTPError as exc: + raise CollectionError(f'GitHub HTTP {exc.code}') from None + except (OSError, ValueError): + raise CollectionError('GitHub unavailable') from None + + def pages(self, path, params): + rows = [] + for page in range(1, MAX_PAGES + 1): + data = self.get(path + '?' + urlencode(dict(params, per_page=100, page=page))) + if not isinstance(data, list): + raise CollectionError('invalid response') + rows.extend(data) + if len(data) < 100: + return rows + error = CollectionError('pagination limit reached') + error.rows = rows + raise error + + +def closing_issue(body): + """Only one explicit, same-repository closing directive; no fuzzy pairing.""" + body = re.sub(r'```.*?```', '', body or '', flags=re.S) + refs = set(re.findall(r'^\s*(?:Closes|Fixes|Resolves)\s+#(\d+)\s*\.?\s*$', body, re.M | re.I)) + return int(next(iter(refs))) if len(refs) == 1 else None + + +def event(repo, number, suffix, when, kind, summary, title, url, now): + dt = instant(when) + if dt > now + timedelta(minutes=5): + raise ValueError('future timestamp') + expected = f'https://github.com/{repo}/' + if not isinstance(url, str) or not url.startswith(expected): + raise ValueError('invalid source URL') + return {'id': f'{repo}#{number}:{suffix}', 'task': f'{repo}#{number}', + 'repo': repo, 'time': stamp(dt), 'kind': kind, 'summary': short(summary), + 'title': short(title), 'url': url} + + +def collect_repo(api, repo, since, now, prior): + """Transactional per-source batch: a failed page leaves cursor unchanged.""" + partial = [] + def page_set(path, params): + try: + return api.pages(path, params) + except CollectionError as exc: + if hasattr(exc, 'rows'): + partial.append(str(exc)) + return exc.rows + raise + issues = page_set(f'/repos/{repo}/issues', {'state': 'all', 'sort': 'updated', 'direction': 'asc', 'since': since}) + comments = page_set(f'/repos/{repo}/issues/comments', {'sort': 'updated', 'direction': 'asc', 'since': since}) + tasks = dict(prior.get('tasks', {})) + events = dict(prior.get('events', {})) + for item in issues: + n = int(item['number']) + is_pr = 'pull_request' in item + # Resuming a bounded initial scan must not spend the entire next budget + # re-fetching PR details already normalized from this exact revision. + if tasks.get(str(n), {}).get('revision') == item.get('updated_at') and item.get('updated_at'): + continue + try: + detail = api.get(f'/repos/{repo}/pulls/{n}') if is_pr else item + except CollectionError as exc: + partial.append(str(exc)) + break + link = closing_issue(detail.get('body')) if is_pr else None + tasks[str(n)] = {'title': short(item['title']), 'is_pr': is_pr, 'issue': link, 'state': item['state'], 'revision': item.get('updated_at')} + url = f'https://github.com/{repo}/{"pull" if is_pr else "issues"}/{n}' + e = event(repo, n, 'opened', item['created_at'], 'opened', 'Draft PR opened' if detail.get('draft') else 'PR opened' if is_pr else 'Task opened', item['title'], url, now) + events[e['id']] = e + if detail.get('merged_at'): + e = event(repo, n, 'merged', detail['merged_at'], 'merged', 'PR merged', item['title'], url, now) + events[e['id']] = e + # A previously observed closed state is replaced by precise merge evidence. + events.pop(f'{repo}#{n}:closed', None) + elif item.get('closed_at'): + e = event(repo, n, 'closed', item['closed_at'], 'closed', 'PR closed without merge' if is_pr else 'Task closed', item['title'], url, now) + events[e['id']] = e + elif item.get('state') == 'open': + # We cannot timestamp a reopening from updated_at; keep historical events. + pass + for comment in comments: + if comment.get('user', {}).get('type') == 'Bot': + continue + match = re.fullmatch(r'https://api.github.com/repos/' + re.escape(repo) + r'/issues/(\d+)', comment.get('issue_url', '')) + if not match: + continue + n = match[1] + if n not in tasks: + partial.append('comment task metadata not yet collected') + continue + body = comment.get('body') or '' + heading = re.search(r'^#{1,3}\s+(Progress(?: update)?|In progress|Blocked|Shipped|Workspace PR Created|Library PR Created)\b', body, re.M | re.I) + stage = heading.group(1).lower() if heading else '' + kind, summary = {'blocked': ('blocked', 'Blocker reported'), 'in progress': ('active', 'Work started'), + 'shipped': ('reported', 'Shipment reported — see source'), + 'workspace pr created': ('progress', 'Workspace PR recorded'), + 'library pr created': ('progress', 'Library PR recorded')}.get(stage, ('progress', 'Progress update recorded' if heading else 'Discussion update recorded')) + # Comment edits replace this event in-place at its original posted time. + # No comment text is ever exported or interpreted as scientific acceptance. + e = event(repo, int(n), 'comment-' + str(int(comment['id'])), comment['created_at'], kind, summary, + tasks[n]['title'], comment['html_url'], now) + events[e['id']] = e + cutoff = now - timedelta(days=RETENTION_DAYS) + recent = sorted((e for e in events.values() if instant(e['time']) >= cutoff), key=lambda e: (e['time'], {'merged': 5, 'closed': 4, 'blocked': 3, 'progress': 2}.get(e['kind'], 1), e['id']), reverse=True) + truncated = len(recent) > MAX_EVENTS + recent = recent[:MAX_EVENTS] + keep = {e['task'].rsplit('#', 1)[1] for e in recent} + keep.update(str(tasks[n]['issue']) for n in list(keep) if tasks.get(n, {}).get('issue')) + tasks = {n: t for n, t in tasks.items() if n in keep} + if truncated: + partial.append('event retention cap reached') + result = {'events': {e['id']: e for e in recent}, 'tasks': tasks, + 'checked': stamp(now), 'status': 'partial' if partial else 'ok', + 'reason': '; '.join(dict.fromkeys(partial)) or None} + if not partial: + result['cursor'] = stamp(now) + elif prior.get('cursor'): + result['cursor'] = prior['cursor'] + return result + + +def collect(registry, previous, api, now=None): + now = now or datetime.now(timezone.utc) + previous = previous if previous.get('schema_version') == SCHEMA else {} + cache = {'schema_version': SCHEMA, 'sources': {}, 'next': 0} + names = list(registry) + offset = int(previous.get('next', 0)) % max(1, len(names)) + ordered = names[offset:] + names[:offset] + # Public catalogs avoid one visibility request per repository on every tick. + # A missing/failed owner catalog removes its entries from the exported cache. + catalogs = {} + for owner in sorted({repo.split('/')[0] for repo in names}): + try: + identity = api.get('/users/' + owner) + route = '/orgs/' if identity.get('type') == 'Organization' else '/users/' + entries = api.pages(route + owner + '/repos', {'type': 'public' if route == '/orgs/' else 'owner', 'sort': 'full_name'}) + catalogs[owner] = {item['full_name'].lower() for item in entries + if item.get('private') is False and item.get('visibility', 'public') == 'public'} + except (CollectionError, ValueError, KeyError, TypeError): + catalogs[owner] = set() + public = set() + for repo in ordered: + old = previous.get('sources', {}).get(repo, {}) + if repo.lower() in catalogs[repo.split('/')[0]]: + public.add(repo) + cache['sources'][repo] = dict(old, status='stale', reason='awaiting collection') + else: + cache['sources'][repo] = {'status': 'unavailable', 'reason': 'public source unavailable', 'events': {}, 'tasks': {}} + advanced = 0 + for repo in ordered: + if repo not in public: + advanced += 1 + continue + old = cache['sources'][repo] + since = stamp(max(now - timedelta(days=RETENTION_DAYS), instant(old['cursor']) - timedelta(minutes=2))) if old.get('cursor') else stamp(now - timedelta(days=7)) + try: + cache['sources'][repo] = collect_repo(api, repo, since, now, old) + advanced += 1 + except (CollectionError, ValueError, KeyError, TypeError) as exc: + old['status'] = 'stale' + old['reason'] = str(exc) if isinstance(exc, CollectionError) else 'invalid source data' + if api.requests >= api.budget or getattr(api, 'expired', False): + break + advanced += 1 + cache['next'] = (offset + max(1, advanced)) % max(1, len(names)) + feed = make_feed(registry, cache, now) + return cache, feed + + +def make_feed(registry, cache, now): + grouped = {} + coverage = [] + cutoff = now - timedelta(days=RETENTION_DAYS) + for repo, source in cache['sources'].items(): + coverage.append({'repo': repo, 'status': source['status'], 'checked': source.get('checked'), 'reason': source.get('reason')}) + tasks = source.get('tasks', {}) + for e in source.get('events', {}).values(): + if instant(e['time']) < cutoff: + continue + n = e['task'].rsplit('#', 1)[1] + task = tasks.get(n, {}) + target = str(task.get('issue')) + # Only group when the explicit target is present and is actually an issue. + if target in tasks and not tasks[target]['is_pr']: + n, task = target, tasks[target] + key = f'{repo}#{n}' + row = grouped.setdefault(key, {'id': key, 'repo': repo, **registry[repo], 'title': task.get('title', e['title']), + 'state': task.get('state', 'unknown'), 'source_status': source['status'], 'events': []}) + row['events'].append(e) + for row in grouped.values(): + row['events'].sort(key=lambda e: (e['time'], {'merged': 5, 'closed': 4, 'blocked': 3, 'progress': 2}.get(e['kind'], 1), e['id']), reverse=True) + row['history_count'] = len(row['events']) + row['events'] = row['events'][:20] + row['time'] = row['events'][0]['time'] + rows = sorted(grouped.values(), key=lambda row: (row['time'], row['id']), reverse=True) + return {'schema_version': SCHEMA, 'generated': stamp(now), 'scope': 'Recorded development activity', + 'retention_days': RETENTION_DAYS, 'initial_backfill_days': 7, 'truncated': len(rows) > MAX_TASKS, + 'coverage': coverage, 'tasks': rows[:MAX_TASKS]} + + +CSS = ''' +.live-feed{margin:1rem 0 2rem;border:1px solid var(--line);border-radius:12px;padding:1rem} +.feed-toolbar{display:flex;flex-wrap:wrap;align-items:center;justify-content:space-between;gap:.5rem} +.feed-toolbar p,.feed-meta{color:var(--muted);font-size:.85rem;margin:.25rem 0} +.feed-list{list-style:none;padding:0;margin:.8rem 0} +.feed-list>li{margin:0} +.feed-task{margin:0;border-top:1px solid var(--line);padding:.35rem 0;min-width:0} +.feed-task summary{padding:.25rem 0;cursor:pointer;display:flex;align-items:baseline;gap:.6rem;flex-wrap:wrap;list-style:none} +.feed-task summary::before{content:'▸';color:var(--muted)} +.feed-task[open] summary::before{content:'▾'} +.feed-title{flex:1 1 18rem;min-width:0;overflow-wrap:anywhere;font-weight:600} +.feed-time,.feed-stage{font-size:.85rem;color:var(--muted)} +.feed-board{font-size:.85rem;font-weight:600;text-decoration:none} +.feed-event{margin:.45rem 0;overflow-wrap:anywhere;font-size:.9rem} +.feed-event time{color:var(--muted);margin-right:.5rem} +.feed-task ol{padding-left:1.5rem} +.live-feed button{min-height:40px;padding:.4rem .8rem;border:1px solid var(--line);border-radius:6px;background:var(--bg);color:var(--fg);cursor:pointer} +.live-feed :focus-visible{outline:3px solid var(--accent);outline-offset:3px} +.feed-coverage{margin-top:.75rem;font-size:.85rem;color:var(--muted)} +.feed-coverage ul{max-height:14rem;overflow:auto;overflow-wrap:anywhere} +@media(max-width:40rem){.feed-title{flex-basis:100%}.feed-task summary{gap:.35rem}.live-feed{padding:.75rem}} +''' + + +def render_html(feed=None): + feed = feed or {'schema_version': SCHEMA, 'tasks': [], 'coverage': [], 'generated': None} + esc = html.escape + rows = [] + for row in feed['tasks'][:5]: + e = row['events'][0] + rows.append(f'
  • {esc(row["title"])}{esc(e["summary"])}{esc(BOARDS[row["board"]])} · Source
    • ') + data = json.dumps(feed, ensure_ascii=False).replace('<', '\\u003c').replace('>', '\\u003e').replace('&', '\\u0026') + return f'''

    Live Feed

    +
    +

    Recorded development activity

    Collection {esc(feed.get('generated') or 'not yet available')}

    +

    +

    No recorded activity available yet.

    + +
    Coverage and freshness

    Public recorded development only; scientific runs and unrecorded local work are not included. Up to 30 days retained; initial collection looks back seven days. Collection runs approximately every five minutes; repositories share a bounded request budget.

      + +
      ''' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--repos', required=True, type=Path) + ap.add_argument('--cache', required=True, type=Path) + ap.add_argument('--output', required=True, type=Path) + ap.add_argument('--budget', type=int, default=60) + args = ap.parse_args() + # 180 calls / 5min fits a 5000/hour PAT. Default GITHUB_TOKEN uses 60/cycle. + previous = json.loads(args.cache.read_text()) if args.cache.exists() else {} + cache, feed = collect(sources(args.repos), previous, GitHub(os.environ.get('GH_TOKEN'), args.budget)) + for path, value in ((args.cache, cache), (args.output, feed)): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n') + print(json.dumps({'tasks': len(feed['tasks']), 'sources': len(feed['coverage']), 'current': sum(c['status'] == 'ok' for c in feed['coverage'])})) + + +if __name__ == '__main__': + main() diff --git a/scripts/live_feed.js b/scripts/live_feed.js new file mode 100644 index 0000000..f5fbddf --- /dev/null +++ b/scripts/live_feed.js @@ -0,0 +1,109 @@ +/* Static, public evidence only. Updating the feed must not move a reader. */ +(() => { +'use strict'; +const root = document.getElementById('live-feed'); +if (!root) return; +const list = root.querySelector('[data-feed-list]'); +const more = root.querySelector('[data-feed-more]'); +const fresh = root.querySelector('[data-feed-freshness]'); +const notice = root.querySelector('[data-feed-notice]'); +const incoming = root.querySelector('[data-feed-new]'); +const empty = root.querySelector('[data-feed-empty]'); +const coverage = root.querySelector('[data-feed-coverage]'); +let current, pending, count = 5, refreshing = false, lastChecked = null; +function safeURL(value, host) { + try { const u = new URL(value); return u.protocol === 'https:' && !u.username && !u.password && (host ? u.hostname === host : u.hostname.endsWith('.github.io')); } catch (_) { return false; } +} +function validate(feed) { + if (!feed || feed.schema_version !== 1 || !Array.isArray(feed.tasks) || feed.tasks.length > 200 || !Array.isArray(feed.coverage)) throw Error('Invalid activity feed'); + if (feed.generated !== null && !Number.isFinite(Date.parse(feed.generated))) throw Error('Invalid collection time'); + const ids = new Set(); + for (const t of feed.tasks) { + if (typeof t.id !== 'string' || ids.has(t.id) || typeof t.title !== 'string' || typeof t.board !== 'string' || typeof t.repo !== 'string' || !safeURL(t.board_url) || !Array.isArray(t.events) || !t.events.length || t.events.length > 20) throw Error('Invalid task'); + ids.add(t.id); + for (const e of t.events) if (typeof e.id !== 'string' || typeof e.summary !== 'string' || !Number.isFinite(Date.parse(e.time)) || !safeURL(e.url, 'github.com')) throw Error('Invalid event'); + } + return feed; +} +function el(tag, text, className) { + const node = document.createElement(tag); if (text !== undefined) node.textContent = text; if (className) node.className = className; return node; +} +function link(text, href, className) { const a = el('a', text, className); a.href = href; return a; } +function age(value) { + const mins = Math.max(0, Math.floor((Date.now() - Date.parse(value)) / 60000)); + return mins < 1 ? 'just now' : mins < 60 ? `${mins}m ago` : mins < 1440 ? `${Math.floor(mins / 60)}h ago` : `${Math.floor(mins / 1440)}d ago`; +} +function time(value) { const node = el('time', age(value), 'feed-time'); node.dateTime = value; node.title = new Date(value).toLocaleString(); return node; } +function taskRow(task) { + const li = el('li'); const details = el('details', undefined, 'feed-task'); details.dataset.task = task.id; + const summary = el('summary'); const latest = task.events[0]; + summary.append(link(({scientist:'🔭',brain:'🧠',mind:'💭',cortex:'🔬',heart:'❤️',hands:'🛠️',pulse:'⏱️',insight:'💡',dna:'🧬',memory:'📚',eyes:'👁️',ears:'👂',broca:'🗣️',nerves:'⚡',gut:'♻️'}[task.board] || '') + ' ' + task.board.replace(/^./, c => c.toUpperCase()), task.board_url, 'feed-board'), el('span', task.title, 'feed-title'), el('span', latest.summary, 'feed-stage'), time(latest.time)); + details.append(summary, el('p', `${task.repo} · ${task.state || 'unknown'}${task.source_status !== 'ok' ? ' · last known evidence' : ''}`, 'feed-meta')); + const history = el('ol'); + for (const e of task.events) { const item = el('li', undefined, 'feed-event'); item.append(time(e.time), document.createTextNode(' '), link(e.summary, e.url)); history.append(item); } + details.append(history); + if (task.history_count > task.events.length) details.append(el('p', 'Showing the latest 20 recorded updates. Open the source for older history.', 'feed-meta')); + li.append(details); return li; +} +function renderTasks() { + const opened = new Set([...list.querySelectorAll('details[open]')].map(x => x.dataset.task)); + const rows = current.tasks.slice(0, count).map(task => { const row = taskRow(task); if (opened.has(task.id)) row.firstElementChild.open = true; return row; }); + list.replaceChildren(...rows); + more.hidden = count >= current.tasks.length; + empty.hidden = current.tasks.length > 0; + empty.textContent = !current.generated || !current.coverage.length ? 'Activity collection not yet available.' : current.coverage.some(c => c.status !== 'ok') ? 'No activity available from the checked sources. See coverage below.' : 'No recorded development activity in the collected window.'; +} +function renderCoverage() { + const gaps = current.coverage.filter(c => c.status !== 'ok').length; + const aged = current.generated && Date.now() - Date.parse(current.generated) > 15 * 60000; + fresh.textContent = current.generated ? `Collected ${age(current.generated)}${aged ? ' · delayed' : ''}${lastChecked ? ' · checked ' + age(lastChecked) : ''}` : 'Collection not yet available'; + root.querySelector('[data-feed-coverage-label]').textContent = `Coverage and freshness · ${current.coverage.length - gaps}/${current.coverage.length} sources checked${gaps ? ' · gaps' : ''}${current.truncated ? ' · latest 200 tasks' : ''}`; + coverage.replaceChildren(...current.coverage.map(c => el('li', `${c.repo}: ${c.status}${c.checked ? ' · checked ' + age(c.checked) : ''}${c.reason ? ' · ' + c.reason : ''}`))); +} +function reading() { + return count > 5 || Boolean(root.querySelector('details[open]')) || root.contains(document.activeElement) || root.getBoundingClientRect().top < -30; +} +function signature(feed) { return JSON.stringify(feed.tasks); } +function apply(feed) { current = feed; pending = null; incoming.hidden = true; renderTasks(); renderCoverage(); } +function receive(feed) { + if ((pending?.generated || current.generated) && feed.generated && Date.parse(feed.generated) < Date.parse(pending?.generated || current.generated)) throw Error('Older feed'); + if (signature(feed) !== signature(current)) { + if (reading()) { + pending = feed; + const prior = new Map(current.tasks.map(t => [t.id, JSON.stringify(t)])); + const changed = feed.tasks.filter(t => prior.get(t.id) !== JSON.stringify(t)).length; + incoming.textContent = changed ? `${changed} updated task${changed === 1 ? '' : 's'} · Show updates` : 'Feed changed · Show updates'; + incoming.hidden = false; + } else apply(feed); + } else { current = {...feed, tasks: current.tasks}; pending = null; incoming.hidden = true; renderCoverage(); } +} +more.addEventListener('click', () => { + // Append; existing details and their focus remain untouched. + const start = count; count += 5; + for (const task of current.tasks.slice(start, count)) list.append(taskRow(task)); + more.hidden = count >= current.tasks.length; + if (more.hidden) list.lastElementChild?.querySelector('summary')?.focus(); +}); +incoming.addEventListener('click', () => { + if (!pending) return; + const top = root.getBoundingClientRect().top; + apply(pending); + const first = list.querySelector('summary'); if (first) first.focus({preventScroll:true}); + window.scrollBy(0, root.getBoundingClientRect().top - top); +}); +async function refresh() { + if (refreshing || document.visibilityState === 'hidden') return; + refreshing = true; + const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 15000); + try { + const response = await fetch(new URL('activity.json', location.href), {cache:'no-store', signal:controller.signal}); + if (!response.ok) throw Error('Activity unavailable'); + const feed = validate(await response.json()); lastChecked = new Date().toISOString(); receive(feed); notice.textContent = ''; + } catch (_) { notice.textContent = 'Live updates unavailable. Showing the last loaded evidence.'; } + finally { clearTimeout(timeout); refreshing = false; } +} +try { current = validate(JSON.parse(document.getElementById('live-feed-snapshot').textContent)); apply(current); } +catch (_) { notice.textContent = 'Activity snapshot unavailable.'; return; } +refresh(); setInterval(refresh, 60000); +document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'visible') refresh(); }); +})(); diff --git a/scripts/organism_board.py b/scripts/organism_board.py index 890596d..a67e288 100644 --- a/scripts/organism_board.py +++ b/scripts/organism_board.py @@ -331,6 +331,12 @@ def _render_html(snapshot: dict) -> str: {organ_name}{lifecycle}
      Published status
      {head}
      {_html.escape(note)}Open {organ_name} →
      """) + activity_html = activity_css = activity_script = "" + if instance is None: + from activity_feed import render_html as render_activity, CSS as ACTIVITY_CSS + activity_html = render_activity(snapshot.get("activity")) + activity_css = ACTIVITY_CSS + activity_script = (HOME / "scripts" / "live_feed.js").read_text(encoding="utf-8") catalogue = "" navigation = views = navigation_css = navigation_script = "" if instance is not None: @@ -357,9 +363,10 @@ def _render_html(snapshot: dict) -> str: # whole grid inside section_layout's extra collapsed H2 group. return f""" -{_html.escape(name)} Dashboard +{_html.escape(name)} Dashboard {navigation}{'
      ' if instance is not None else ""}{'
      ' if instance is not None else ""}{banner}{panel} {projects_html} +{activity_html}

      Dashboards

      {'

      No organs connected. Add a connection when your project needs it.

      ' if instance is not None and not snapshot.get("boards") else ''}
      {''.join(rows)}
      @@ -369,7 +376,7 @@ def _render_html(snapshot: dict) -> str: {'
      ' if instance is not None else ""}{views}{'
      ' if instance is not None else ""} {'' if instance is not None else ""} -""" +""" def badge_endpoint(snapshot: dict) -> dict: @@ -410,6 +417,7 @@ def main(argv: list[str] | None = None) -> int: g.add_argument("--badge", action="store_true") g.add_argument("--json", action="store_true") g.add_argument("--catalogue", action="store_true", help="export public resource relationships offline; no feed collection") + ap.add_argument("--activity", type=Path, help="public normalized activity snapshot (canonical Scientist only)") ap.add_argument("--instance", type=Path, help="opt-in instance profile; selects only explicitly connected organs") ap.add_argument("--projects", type=Path, help="instance-owned project/resource configuration") @@ -448,6 +456,8 @@ def main(argv: list[str] | None = None) -> int: return 0 snap = collect(instance=instance) snap["project_resources"] = projects + if ns.activity and instance is None: + snap["activity"] = json.loads(ns.activity.read_text()) fmt = "md" for name, label in (("md", "md"), ("md_brief", "md-brief"), ("html", "html"), ("badge", "badge"), ("json", "json")): diff --git a/tests/live_feed_browser.cjs b/tests/live_feed_browser.cjs new file mode 100644 index 0000000..f143dda --- /dev/null +++ b/tests/live_feed_browser.cjs @@ -0,0 +1,58 @@ +/* Optional real-browser regression: node tests/live_feed_browser.cjs . */ +const fs = require('node:fs'); +const path = require('node:path'); +const assert = require('node:assert/strict'); +const {chromium} = require('playwright'); +(async () => { + const [fixtures, output] = process.argv.slice(2); + fs.mkdirSync(output, {recursive:true}); + const html = fs.readFileSync(path.join(fixtures,'browser-preview.html'),'utf8'); + const original = JSON.parse(fs.readFileSync(path.join(fixtures,'browser-feed.json'),'utf8')); + const browser = await chromium.launch({executablePath:process.env.PLAYWRIGHT_EXECUTABLE_PATH,headless:true,args:['--no-sandbox']}); + try { + for (const width of [390,768,1440]) for (const colorScheme of ['light','dark']) { + const context = await browser.newContext({viewport:{width,height:1100},colorScheme}); + const page = await context.newPage(); const errors=[]; page.on('pageerror',e=>errors.push(e.message)); + let feed = structuredClone(original); let requests=0; + await page.route('**/*',route=> { + if (route.request().url().includes('activity.json')) {requests++;return route.fulfill({contentType:'application/json',body:JSON.stringify(feed)});} + return route.fulfill({contentType:'text/html',body:html}); + }); + await page.goto('https://preview.example/'); + await page.waitForFunction(()=>document.querySelector('[data-feed-freshness]').textContent.includes('checked')); + assert.equal(await page.locator('.feed-task').count(),5); + assert.equal(await page.evaluate(()=>document.documentElement.scrollWidth>innerWidth),false); + const first = page.locator('.feed-task').first(); + await first.locator('summary').click(); + assert.equal(await first.getAttribute('open'),''); + const oldText = await first.locator('.feed-title').innerText(); + const oldId = await first.getAttribute('data-task'); + feed.tasks[0].title='A newer task title';feed.generated='2026-10-13T20:01:00Z'; + await page.evaluate(()=>document.dispatchEvent(new Event('visibilitychange'))); + await page.locator('[data-feed-new]').waitFor({state:'visible'}); + assert.equal(await first.locator('.feed-title').innerText(),oldText); + assert.equal(await first.getAttribute('open'),''); + await page.locator('[data-feed-new]').click(); + assert.equal(await page.locator('.feed-task').first().locator('.feed-title').innerText(),'A newer task title'); + assert.equal(await page.locator(`details[data-task="${oldId}"]`).getAttribute('open'),''); + await page.locator('[data-feed-more]').click(); + assert.equal(await page.locator('.feed-task').count(),8); + assert.equal(await page.locator(`details[data-task="${oldId}"]`).getAttribute('open'),''); + // Unsafe response must leave the last good snapshot visible. + feed.tasks[0].events[0].url='javascript:alert(1)'; + await page.evaluate(()=>document.dispatchEvent(new Event('visibilitychange'))); + await page.waitForFunction(()=>document.querySelector('[data-feed-notice]').textContent.includes('unavailable')); + assert.equal(await page.locator('.feed-task').count(),8); + assert.deepEqual(errors,[]); + assert.equal(await page.evaluate(()=>document.documentElement.scrollWidth>innerWidth),false); + await page.locator('#live-feed-heading').scrollIntoViewIfNeeded(); + await page.screenshot({path:path.join(output,`feed-${width}-${colorScheme}.png`)}); + assert(requests>=3); + await context.close(); + } + const context=await browser.newContext({javaScriptEnabled:false});const page=await context.newPage(); + await page.route('**/*',r=>r.fulfill({contentType:'text/html',body:html}));await page.goto('https://preview.example/'); + assert.equal(await page.locator('.feed-task').count(),5);assert(await page.locator('.feed-task a').first().getAttribute('href'));await context.close(); + console.log('PASS: six viewport/theme cases; grouped history, buffered updates, Show more, unsafe-feed fallback, no overflow/errors; no-JS snapshot.'); + } finally {await browser.close();} +})().catch(e=>{console.error(e);process.exit(1);}); diff --git a/tests/test_activity_feed.py b/tests/test_activity_feed.py new file mode 100644 index 0000000..4a389dd --- /dev/null +++ b/tests/test_activity_feed.py @@ -0,0 +1,182 @@ +"""Offline evidence/coverage tests; no GitHub or scientific execution.""" +import copy +from datetime import datetime, timezone +import json +from pathlib import Path +import sys + +import pytest +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) +import activity_feed as af + +NOW = datetime(2026, 10, 13, 20, tzinfo=timezone.utc) +REPO = 'Example/PyAutoScientist' +REGISTRY = {REPO: {'board': 'scientist', 'board_url': 'https://example.github.io/PyAutoScientist/'}} +TIME = '2026-10-13T19:00:00Z' + + +def issue(n=1, **changes): + return dict(number=n, title='A small task', created_at=TIME, updated_at=TIME, + state='open', closed_at=None, body='', **changes) + + +def comment(body='## Progress\nPrivate-looking long transcript must never be copied', **changes): + d = dict(id=9, body=body, created_at='2026-10-13T19:05:00Z', updated_at='2026-10-13T19:05:00Z', + issue_url=f'https://api.github.com/repos/{REPO}/issues/1', html_url=f'https://github.com/{REPO}/issues/1#issuecomment-9', user={'type': 'User'}) + d.update(changes) + return d + + +class Fake: + def __init__(self, issues=None, comments=None, pulls=None, private=False, fail=False, budget=100): + self.issues, self.comments, self.pulls = issues or [], comments or [], pulls or {} + self.private, self.fail = private, fail + self.requests, self.budget = 0, budget + + def get(self, path): + self.requests += 1 + if '/users/' in path: + return {'type': 'Organization'} + if '/pulls/' in path: + return self.pulls[int(path.rsplit('/', 1)[1])] + raise AssertionError(path) + + def pages(self, path, params): + self.requests += 1 + if path == '/orgs/Example/repos': + return [{'full_name': REPO, 'private': self.private}] + if self.fail: + raise af.CollectionError('GitHub unavailable') + return copy.deepcopy(self.comments if path.endswith('/comments') else self.issues) + + +def test_task_pr_grouped_but_not_fuzzy_title_match(): + pr = issue(2, pull_request={}) + pr3 = issue(3, pull_request={}) + api = Fake([issue(), pr, pr3], pulls={2: {'body': 'Closes #1', 'merged_at': '2026-10-13T19:20:00Z'}, 3: {'body': 'Similar task #1'}}) + _, feed = af.collect(REGISTRY, {}, api, NOW) + assert len(feed['tasks']) == 2 + task = next(t for t in feed['tasks'] if t['id'].endswith('#1')) + assert task['events'][0]['summary'] == 'PR merged' + assert len(task['events']) == 3 + + +def test_comment_edits_not_new_events_and_no_raw_text_export(): + cache, _ = af.collect(REGISTRY, {}, Fake([issue()], [comment()]), NOW) + cache, feed = af.collect(REGISTRY, cache, Fake([issue()], [comment(body='## Blocked\nsecretish text', updated_at=TIME)]), NOW) + events = feed['tasks'][0]['events'] + assert len(events) == 2 + assert events[0]['summary'] == 'Blocker reported' + assert events[0]['time'] == '2026-10-13T19:05:00Z' + assert 'secretish' not in json.dumps(cache) + assert 'transcript' not in json.dumps(cache) + + +def test_refresh_is_not_an_event_and_no_fake_completion(): + cache, feed = af.collect(REGISTRY, {}, Fake([issue()]), NOW) + _, newer = af.collect(REGISTRY, cache, Fake(), NOW) + assert newer['tasks'][0]['events'] == feed['tasks'][0]['events'] + assert newer['tasks'][0]['events'][0]['summary'] == 'Task opened' + + +def test_failed_source_retains_last_known_without_advancing_cursor(): + cache, _ = af.collect(REGISTRY, {}, Fake([issue()]), NOW) + old_cursor = cache['sources'][REPO]['cursor'] + cache, feed = af.collect(REGISTRY, cache, Fake(fail=True), NOW) + assert cache['sources'][REPO]['cursor'] == old_cursor + assert feed['tasks'][0]['source_status'] == 'stale' + assert feed['coverage'][0]['status'] == 'stale' + + +def test_private_source_never_exports_even_cached_content(): + cache, _ = af.collect(REGISTRY, {}, Fake([issue()]), NOW) + cache, feed = af.collect(REGISTRY, cache, Fake(private=True), NOW) + assert not feed['tasks'] + assert 'A small task' not in json.dumps(cache) + assert feed['coverage'][0]['status'] == 'unavailable' + + +def test_closed_unmerged_is_not_merged(): + pr = issue(2, pull_request={}); pr.update(closed_at=TIME, state='closed') + _, feed = af.collect(REGISTRY, {}, Fake([pr], pulls={2: {}}), NOW) + assert {e['summary'] for e in feed['tasks'][0]['events']} == {'PR opened', 'PR closed without merge'} + + +def test_bot_noise_omitted_and_unstructured_comment_link_only(): + bot = comment(user={'type': 'Bot'}, id=10) + _, feed = af.collect(REGISTRY, {}, Fake([issue()], [comment('Hello ' + _, feed = af.collect(REGISTRY, {}, Fake([bad]), NOW) + rendered = af.render_html(feed) + assert '