From 229e603a974a16e25edc50f2a39749b520d95f81 Mon Sep 17 00:00:00 2001 From: "aidan.casey" Date: Sun, 6 Sep 2026 14:09:59 -0400 Subject: [PATCH] Testing --- .github/workflows/ai-detection.yml | 545 +---------------------------- 1 file changed, 12 insertions(+), 533 deletions(-) diff --git a/.github/workflows/ai-detection.yml b/.github/workflows/ai-detection.yml index 78f4b5f1b..fcfc35799 100644 --- a/.github/workflows/ai-detection.yml +++ b/.github/workflows/ai-detection.yml @@ -368,547 +368,26 @@ jobs: cache: true prompt: ${{ steps.prompt.outputs.content }} - - name: Validate model response and calculate score - id: score + - name: Output raw model response env: MODEL_RESPONSE: ${{ steps.model.outputs.response }} shell: bash run: | - set -euo pipefail - - python3 <<'PY' - import json - import os - from pathlib import Path - - raw = os.environ.get( - "MODEL_RESPONSE", - "", - ).strip() - - if not raw: - raise SystemExit( - "Ollama returned an empty response." - ) - - # Small models occasionally surround valid JSON with a short - # preamble despite being instructed otherwise. - start = raw.find("{") - end = raw.rfind("}") - - if ( - start == -1 - or end == -1 - or end <= start - ): - print("Raw model response:") - print(raw[:4000]) - - raise SystemExit( - "Could not locate JSON in model response." - ) - - try: - result = json.loads( - raw[start:end + 1] - ) - except json.JSONDecodeError as exc: - print("Raw model response:") - print(raw[:4000]) - raise SystemExit( - f"Invalid JSON from model: {exc}" - ) - - def score(value): - try: - value = int(value) - except ( - ValueError, - TypeError, - ): - value = 0 - - return max( - 0, - min(100, value), - ) - - def signals(value): - if not isinstance( - value, - list, - ): - return [] - - cleaned = [] - - for item in value[:5]: - if not isinstance( - item, - str, - ): - continue - - item = ( - item.strip() - .replace("\n", " ") - ) - - if item: - cleaned.append( - item[:300] - ) - - return cleaned - - metadata = result.get( - "metadata", - {}, - ) - - code = result.get( - "code", - {}, - ) - - metadata_likelihood = score( - metadata.get( - "likelihood", - 0, - ) - ) - - metadata_confidence = score( - metadata.get( - "confidence", - 0, - ) - ) - - code_likelihood = score( - code.get( - "likelihood", - 0, - ) - ) - - code_confidence = score( - code.get( - "confidence", - 0, - ) - ) - - explicit = ( - result.get( - "explicit_ai_disclosure", - False, - ) - is True - ) - - # Deterministic weighting. - # - # The model supplies evidence assessments but does not control - # your repository policy. - overall = round( - metadata_likelihood * 0.30 - + code_likelihood * 0.70 - ) - - overall_confidence = round( - metadata_confidence * 0.30 - + code_confidence * 0.70 - ) - - # Direct disclosure is much stronger evidence than stylistic - # inference from code. - if explicit: - overall = max( - overall, - 90, - ) - - overall_confidence = max( - overall_confidence, - 85, - ) - - summary = result.get( - "summary", - "", - ) - - if not isinstance( - summary, - str, - ): - summary = "" - - normalized = { - "metadata": { - "likelihood": ( - metadata_likelihood - ), - "confidence": ( - metadata_confidence - ), - "signals_for": signals( - metadata.get( - "signals_for", - [], - ) - ), - "signals_against": signals( - metadata.get( - "signals_against", - [], - ) - ), - }, - "code": { - "likelihood": ( - code_likelihood - ), - "confidence": ( - code_confidence - ), - "signals_for": signals( - code.get( - "signals_for", - [], - ) - ), - "signals_against": signals( - code.get( - "signals_against", - [], - ) - ), - }, - "explicit_ai_disclosure": ( - explicit - ), - "overall_likelihood": ( - overall - ), - "overall_confidence": ( - overall_confidence - ), - "summary": ( - summary.strip()[:600] - ), - } - - Path( - "result.json" - ).write_text( - json.dumps( - normalized, - indent=2, - ), - encoding="utf-8", - ) - - def indication(value): - if value >= 80: - return "Very high indication" - - if value >= 60: - return "High indication" - - if value >= 30: - return "Some indication" - - return "Low indication" - - def confidence(value): - if value >= 75: - return "High" - - if value >= 45: - return "Medium" - - return "Low" - - with open( - os.environ["GITHUB_OUTPUT"], - "a", - encoding="utf-8", - ) as output: - output.write( - f"overall={overall}\n" - ) - output.write( - f"indication={indication(overall)}\n" - ) - output.write( - f"confidence_score={overall_confidence}\n" - ) - output.write( - f"confidence={confidence(overall_confidence)}\n" - ) - output.write( - f"metadata={metadata_likelihood}\n" - ) - output.write( - f"code={code_likelihood}\n" - ) - output.write( - f"explicit={str(explicit).lower()}\n" - ) - PY - - - name: Build PR comment - shell: bash - run: | - set -euo pipefail + echo "==========================================" + echo "RAW OLLAMA RESPONSE" + echo "==========================================" + printf '%s\n' "$MODEL_RESPONSE" + echo "==========================================" - python3 <<'PY' - import json - import textwrap - from pathlib import Path - - result = json.loads( - Path( - "result.json" - ).read_text( - encoding="utf-8" - ) - ) - - pr = json.loads( - Path( - "pr.json" - ).read_text( - encoding="utf-8" - ) - ) - - overall = result[ - "overall_likelihood" - ] - - confidence_score = result[ - "overall_confidence" - ] - - if overall >= 80: - indication = ( - "Very high indication" - ) - elif overall >= 60: - indication = ( - "High indication" - ) - elif overall >= 30: - indication = ( - "Some indication" - ) - else: - indication = ( - "Low indication" - ) - - if confidence_score >= 75: - confidence = "High" - elif confidence_score >= 45: - confidence = "Medium" - else: - confidence = "Low" - - def bullet_list(items): - if not items: - return ( - "_No notable signals identified._" - ) - - return "\n".join( - f"- {item}" - for item in items - ) - - metadata_for = bullet_list( - result["metadata"][ - "signals_for" - ] - ) - - metadata_against = bullet_list( - result["metadata"][ - "signals_against" - ] - ) - - code_for = bullet_list( - result["code"][ - "signals_for" - ] - ) - - code_against = bullet_list( - result["code"][ - "signals_against" - ] - ) - - explicit = ( - "Yes" - if result[ - "explicit_ai_disclosure" - ] - else "No" - ) - - summary = ( - result["summary"] - or ( - "No additional summary " - "was provided." - ) - ) - - changed_files = pr.get( - "changed_files", - "?" - ) - - additions = pr.get( - "additions", - "?" - ) - - deletions = pr.get( - "deletions", - "?" - ) - - comment = f""" - - - ## 🤖 AI-assistance indicators - - **Overall indication:** {overall}% — **{indication}** - **Evidence confidence:** {confidence} ({confidence_score}%) - **PR metadata:** {result["metadata"]["likelihood"]}% - **Code indicators:** {result["code"]["likelihood"]}% - **Explicit AI disclosure:** {explicit} - -
- PR scope - - - Changed files: {changed_files} - - Additions: {additions} - - Deletions: {deletions} - -
- - ### Metadata signals supporting AI assistance - - {metadata_for} - - ### Metadata signals against AI assistance - - {metadata_against} - - ### Code signals supporting AI assistance - - {code_for} - - ### Code signals against AI assistance - - {code_against} - - ### Summary - - {summary} - - --- - - This is a heuristic assessment of **AI-assistance indicators**, - not proof that AI authored the contribution. - - Code quality, grammar, formatting, documentation, or thorough - testing alone should not be interpreted as evidence of AI use. - """ - - Path( - "comment.md" - ).write_text( - textwrap.dedent( - comment - ).strip() - + "\n", - encoding="utf-8", - ) - PY - - cat comment.md - - - name: Create or update analysis comment + - name: Write raw response to job summary env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - shell: bash - run: | - set -euo pipefail - - MARKER='' - - COMMENT_ID="$( - gh api \ - --paginate \ - "/repos/${GH_REPO}/issues/${PR_NUMBER}/comments" \ - --jq \ - ".[] | - select(.body | contains(\"${MARKER}\")) | - .id" \ - | head -n 1 - )" - - BODY="$( - jq -Rs . < comment.md - )" - - if [ -n "${COMMENT_ID}" ]; then - echo \ - "Updating existing comment ${COMMENT_ID}" - - gh api \ - --method PATCH \ - "/repos/${GH_REPO}/issues/comments/${COMMENT_ID}" \ - --input - <> "$GITHUB_STEP_SUMMARY" \ No newline at end of file