diff --git a/.github/workflows/ai-detection.yml b/.github/workflows/ai-detection.yml
index 78f4b5f1b..fcfc35799 100644
--- a/.github/workflows/ai-detection.yml
+++ b/.github/workflows/ai-detection.yml
@@ -368,547 +368,26 @@ jobs:
cache: true
prompt: ${{ steps.prompt.outputs.content }}
- - name: Validate model response and calculate score
- id: score
+ - name: Output raw model response
env:
MODEL_RESPONSE: ${{ steps.model.outputs.response }}
shell: bash
run: |
- set -euo pipefail
-
- python3 <<'PY'
- import json
- import os
- from pathlib import Path
-
- raw = os.environ.get(
- "MODEL_RESPONSE",
- "",
- ).strip()
-
- if not raw:
- raise SystemExit(
- "Ollama returned an empty response."
- )
-
- # Small models occasionally surround valid JSON with a short
- # preamble despite being instructed otherwise.
- start = raw.find("{")
- end = raw.rfind("}")
-
- if (
- start == -1
- or end == -1
- or end <= start
- ):
- print("Raw model response:")
- print(raw[:4000])
-
- raise SystemExit(
- "Could not locate JSON in model response."
- )
-
- try:
- result = json.loads(
- raw[start:end + 1]
- )
- except json.JSONDecodeError as exc:
- print("Raw model response:")
- print(raw[:4000])
- raise SystemExit(
- f"Invalid JSON from model: {exc}"
- )
-
- def score(value):
- try:
- value = int(value)
- except (
- ValueError,
- TypeError,
- ):
- value = 0
-
- return max(
- 0,
- min(100, value),
- )
-
- def signals(value):
- if not isinstance(
- value,
- list,
- ):
- return []
-
- cleaned = []
-
- for item in value[:5]:
- if not isinstance(
- item,
- str,
- ):
- continue
-
- item = (
- item.strip()
- .replace("\n", " ")
- )
-
- if item:
- cleaned.append(
- item[:300]
- )
-
- return cleaned
-
- metadata = result.get(
- "metadata",
- {},
- )
-
- code = result.get(
- "code",
- {},
- )
-
- metadata_likelihood = score(
- metadata.get(
- "likelihood",
- 0,
- )
- )
-
- metadata_confidence = score(
- metadata.get(
- "confidence",
- 0,
- )
- )
-
- code_likelihood = score(
- code.get(
- "likelihood",
- 0,
- )
- )
-
- code_confidence = score(
- code.get(
- "confidence",
- 0,
- )
- )
-
- explicit = (
- result.get(
- "explicit_ai_disclosure",
- False,
- )
- is True
- )
-
- # Deterministic weighting.
- #
- # The model supplies evidence assessments but does not control
- # your repository policy.
- overall = round(
- metadata_likelihood * 0.30
- + code_likelihood * 0.70
- )
-
- overall_confidence = round(
- metadata_confidence * 0.30
- + code_confidence * 0.70
- )
-
- # Direct disclosure is much stronger evidence than stylistic
- # inference from code.
- if explicit:
- overall = max(
- overall,
- 90,
- )
-
- overall_confidence = max(
- overall_confidence,
- 85,
- )
-
- summary = result.get(
- "summary",
- "",
- )
-
- if not isinstance(
- summary,
- str,
- ):
- summary = ""
-
- normalized = {
- "metadata": {
- "likelihood": (
- metadata_likelihood
- ),
- "confidence": (
- metadata_confidence
- ),
- "signals_for": signals(
- metadata.get(
- "signals_for",
- [],
- )
- ),
- "signals_against": signals(
- metadata.get(
- "signals_against",
- [],
- )
- ),
- },
- "code": {
- "likelihood": (
- code_likelihood
- ),
- "confidence": (
- code_confidence
- ),
- "signals_for": signals(
- code.get(
- "signals_for",
- [],
- )
- ),
- "signals_against": signals(
- code.get(
- "signals_against",
- [],
- )
- ),
- },
- "explicit_ai_disclosure": (
- explicit
- ),
- "overall_likelihood": (
- overall
- ),
- "overall_confidence": (
- overall_confidence
- ),
- "summary": (
- summary.strip()[:600]
- ),
- }
-
- Path(
- "result.json"
- ).write_text(
- json.dumps(
- normalized,
- indent=2,
- ),
- encoding="utf-8",
- )
-
- def indication(value):
- if value >= 80:
- return "Very high indication"
-
- if value >= 60:
- return "High indication"
-
- if value >= 30:
- return "Some indication"
-
- return "Low indication"
-
- def confidence(value):
- if value >= 75:
- return "High"
-
- if value >= 45:
- return "Medium"
-
- return "Low"
-
- with open(
- os.environ["GITHUB_OUTPUT"],
- "a",
- encoding="utf-8",
- ) as output:
- output.write(
- f"overall={overall}\n"
- )
- output.write(
- f"indication={indication(overall)}\n"
- )
- output.write(
- f"confidence_score={overall_confidence}\n"
- )
- output.write(
- f"confidence={confidence(overall_confidence)}\n"
- )
- output.write(
- f"metadata={metadata_likelihood}\n"
- )
- output.write(
- f"code={code_likelihood}\n"
- )
- output.write(
- f"explicit={str(explicit).lower()}\n"
- )
- PY
-
- - name: Build PR comment
- shell: bash
- run: |
- set -euo pipefail
+ echo "=========================================="
+ echo "RAW OLLAMA RESPONSE"
+ echo "=========================================="
+ printf '%s\n' "$MODEL_RESPONSE"
+ echo "=========================================="
- python3 <<'PY'
- import json
- import textwrap
- from pathlib import Path
-
- result = json.loads(
- Path(
- "result.json"
- ).read_text(
- encoding="utf-8"
- )
- )
-
- pr = json.loads(
- Path(
- "pr.json"
- ).read_text(
- encoding="utf-8"
- )
- )
-
- overall = result[
- "overall_likelihood"
- ]
-
- confidence_score = result[
- "overall_confidence"
- ]
-
- if overall >= 80:
- indication = (
- "Very high indication"
- )
- elif overall >= 60:
- indication = (
- "High indication"
- )
- elif overall >= 30:
- indication = (
- "Some indication"
- )
- else:
- indication = (
- "Low indication"
- )
-
- if confidence_score >= 75:
- confidence = "High"
- elif confidence_score >= 45:
- confidence = "Medium"
- else:
- confidence = "Low"
-
- def bullet_list(items):
- if not items:
- return (
- "_No notable signals identified._"
- )
-
- return "\n".join(
- f"- {item}"
- for item in items
- )
-
- metadata_for = bullet_list(
- result["metadata"][
- "signals_for"
- ]
- )
-
- metadata_against = bullet_list(
- result["metadata"][
- "signals_against"
- ]
- )
-
- code_for = bullet_list(
- result["code"][
- "signals_for"
- ]
- )
-
- code_against = bullet_list(
- result["code"][
- "signals_against"
- ]
- )
-
- explicit = (
- "Yes"
- if result[
- "explicit_ai_disclosure"
- ]
- else "No"
- )
-
- summary = (
- result["summary"]
- or (
- "No additional summary "
- "was provided."
- )
- )
-
- changed_files = pr.get(
- "changed_files",
- "?"
- )
-
- additions = pr.get(
- "additions",
- "?"
- )
-
- deletions = pr.get(
- "deletions",
- "?"
- )
-
- comment = f"""
-
-
- ## 🤖 AI-assistance indicators
-
- **Overall indication:** {overall}% — **{indication}**
- **Evidence confidence:** {confidence} ({confidence_score}%)
- **PR metadata:** {result["metadata"]["likelihood"]}%
- **Code indicators:** {result["code"]["likelihood"]}%
- **Explicit AI disclosure:** {explicit}
-
-
- PR scope
-
- - Changed files: {changed_files}
- - Additions: {additions}
- - Deletions: {deletions}
-
-
-
- ### Metadata signals supporting AI assistance
-
- {metadata_for}
-
- ### Metadata signals against AI assistance
-
- {metadata_against}
-
- ### Code signals supporting AI assistance
-
- {code_for}
-
- ### Code signals against AI assistance
-
- {code_against}
-
- ### Summary
-
- {summary}
-
- ---
-
- This is a heuristic assessment of **AI-assistance indicators**,
- not proof that AI authored the contribution.
-
- Code quality, grammar, formatting, documentation, or thorough
- testing alone should not be interpreted as evidence of AI use.
- """
-
- Path(
- "comment.md"
- ).write_text(
- textwrap.dedent(
- comment
- ).strip()
- + "\n",
- encoding="utf-8",
- )
- PY
-
- cat comment.md
-
- - name: Create or update analysis comment
+ - name: Write raw response to job summary
env:
- GH_TOKEN: ${{ github.token }}
- GH_REPO: ${{ github.repository }}
- shell: bash
- run: |
- set -euo pipefail
-
- MARKER=''
-
- COMMENT_ID="$(
- gh api \
- --paginate \
- "/repos/${GH_REPO}/issues/${PR_NUMBER}/comments" \
- --jq \
- ".[] |
- select(.body | contains(\"${MARKER}\")) |
- .id" \
- | head -n 1
- )"
-
- BODY="$(
- jq -Rs . < comment.md
- )"
-
- if [ -n "${COMMENT_ID}" ]; then
- echo \
- "Updating existing comment ${COMMENT_ID}"
-
- gh api \
- --method PATCH \
- "/repos/${GH_REPO}/issues/comments/${COMMENT_ID}" \
- --input - <> "$GITHUB_STEP_SUMMARY"
\ No newline at end of file