Python support: transpiled Python on every target, Python servers, Pillow and font-from-data

.github/workflows/developer-guide-docs.yml

@@ -183,11 +183,12 @@ jobs:
           # Through retry.sh, transient failures only, like the demo install below:
           # this is the step that resolves the most from Central, and a single 502
           # on a plugin dependency (maven-shared-utils) failed the whole guide build.
-          # dart-runtime and flutter-runtime: docs/demos/common depends on the Flutter
-          # runtime at this version, which nothing else selected here pulls in; they are
+          # dart-runtime, flutter-runtime and python-runtime: docs/demos/common depends on
+          # the Flutter and Python runtimes at this version, which nothing else selected
+          # here pulls in; they are
           # in the reactor because this job runs on JDK 17 (the flutter-modules profile).
           RETRY_ONLY_MATCHING=transient xvfb-run -a bash scripts/ci/retry.sh mvn -B -ntp -f maven/pom.xml \
-            -pl core,javase,android,css-compiler,codenameone-maven-plugin,backend,backend-test,dart-runtime,flutter-runtime \
+            -pl core,javase,android,css-compiler,codenameone-maven-plugin,backend,backend-test,dart-runtime,flutter-runtime,python-runtime \
             -am install \
             -Plocal-dev-javase \
             -DskipTests \

.github/workflows/gradle-smoke.yml

@@ -162,6 +162,10 @@ jobs:
   # for its version) but never compiles anything, so it stays cheap. The same
   # script's upload-jar and android-source comparisons run again here for free.
   gradle-maven-parity-macos:
+    # The Xcode half of the parity check runs nightly only: hosted macOS runners
+    # are the scarce pool, and the Linux job above already holds every other
+    # target to Maven/Gradle parity on the pull request.
+    if: github.event_name != 'pull_request'
     runs-on: macos-15
     timeout-minutes: 60
     env:

.github/workflows/ios-accessibility-thread.yml

@@ -1,19 +1,18 @@
 name: iOS accessibility thread contract
 
 on:
+  # Nightly at 00:11 UTC on master (nightly-watchdog.yml opens an issue when it
+  # fails), and on a pull request only when this workflow's own tree changes.
+  # IOSNative.m reaches it through the nightly: the check needs a hosted macOS
+  # runner, and a pull request touching the iOS port already queues many.
   workflow_dispatch:
   pull_request:
     branches: [master]
     paths:
-      - 'Ports/iOSPort/nativeSources/IOSNative.m'
-      - 'scripts/ios/accessibility-thread-tests/**'
-      - '.github/workflows/ios-accessibility-thread.yml'
-  push:
-    branches: [master]
-    paths:
-      - 'Ports/iOSPort/nativeSources/IOSNative.m'
       - 'scripts/ios/accessibility-thread-tests/**'
       - '.github/workflows/ios-accessibility-thread.yml'
+  schedule:
+    - cron: '11 0 * * *'
 
 permissions:
   contents: read

.github/workflows/linux-build-run.yml

@@ -247,7 +247,19 @@ jobs:
         run: |
           df -h /
           free -m
-          sudo fallocate -l 12G /cn1-swapfile
+          # Size the swap to the disk this runner actually has: a fixed 12G on a
+          # runner image with little free space left the C build nothing, and the
+          # job died "No space left on device" two minutes into it. Keep at least
+          # 40G for the build and take at most 12G.
+          avail=$(df -BG --output=avail / | tail -1 | tr -dc '0-9')
+          swap=$(( avail - 40 ))
+          [ "$swap" -gt 12 ] && swap=12
+          if [ "$swap" -lt 2 ]; then
+            echo "Only ${avail}G free; not adding swap"
+            exit 0
+          fi
+          echo "Adding ${swap}G swap (${avail}G free)"
+          sudo fallocate -l ${swap}G /cn1-swapfile
           sudo chmod 600 /cn1-swapfile
           sudo mkswap /cn1-swapfile
           sudo swapon /cn1-swapfile

.github/workflows/parparvm-tests.yml

@@ -34,6 +34,16 @@ on:
       - 'maven/project-model/**'
       - 'maven/backend/**'
       - 'maven/backend-test/**'
+      # The Python SQLAlchemy step runs these against the PostgreSQL service.
+      - 'maven/python-runtime/**'
+      - 'maven/python-transpiler/**'
+      - 'scripts/python-backend/sqlalchemy-realdb-test.sh'
+      - 'scripts/python-backend/build.sh'
+      - 'scripts/python-backend/e2e-test.sh'
+      - 'scripts/python-backend/e2e_client.py'
+      - 'scripts/python-backend/samples/**'
+      # sqlalchemy-realdb-test.sh compiles the host file system shims from here.
+      - 'scripts/python-bench/hostfs/**'
       - 'scripts/hellocodenameone/pom.xml'
       - 'scripts/hellocodenameone/backend/**'
       # The core classes the backend compiles as its own (the ORM and the entity
@@ -123,6 +133,16 @@ on:
       - 'maven/project-model/**'
       - 'maven/backend/**'
       - 'maven/backend-test/**'
+      # The Python SQLAlchemy step runs these against the PostgreSQL service.
+      - 'maven/python-runtime/**'
+      - 'maven/python-transpiler/**'
+      - 'scripts/python-backend/sqlalchemy-realdb-test.sh'
+      - 'scripts/python-backend/build.sh'
+      - 'scripts/python-backend/e2e-test.sh'
+      - 'scripts/python-backend/e2e_client.py'
+      - 'scripts/python-backend/samples/**'
+      # sqlalchemy-realdb-test.sh compiles the host file system shims from here.
+      - 'scripts/python-bench/hostfs/**'
       - 'scripts/hellocodenameone/pom.xml'
       - 'scripts/hellocodenameone/backend/**'
       # The core classes the backend compiles as its own (the ORM and the entity
@@ -498,6 +518,29 @@ jobs:
           # the step above just installed from this same checkout.
           CN1SS_BACKEND_SKIP_INSTALL: 1
 
+      # The SQLAlchemy conformance programs against the PostgreSQL service
+      # above: the suite in python-transpiler runs them on SQLite only, so this
+      # is the one place the postgresql dialect and the _cn1db bridge meet a
+      # real server. PostgreSQL and the job's MySQL service; MariaDB is run by
+      # hand. Last of the steps using MySQL: each program drops and recreates
+      # the cn1 database.
+      # python-runtime compiles at release 17, hence JDK 17 for the install.
+      - name: Python SQLAlchemy on PostgreSQL and MySQL
+        run: |
+          (cd maven && bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -pl python-runtime,python-transpiler \
+            install -DskipTests -Dspotbugs.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true \
+            -Dmaven.javadoc.skip=true -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo")
+          # The test transpiles against JavaAPI from the same repository, and
+          # the step that installs it for the native samples runs after this.
+          bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -f vm/pom.xml -pl JavaAPI install -DskipTests \
+            -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo"
+          bash scripts/python-backend/sqlalchemy-realdb-test.sh "$GITHUB_WORKSPACE/.m2-repo"
+        env:
+          JAVA_HOME: ${{ env.JDK_17_HOME }}
+          JAVA17_HOME: ${{ env.JDK_17_HOME }}
+          CN1_SQLA_POSTGRES: postgres://cn1:cn1secret@127.0.0.1:5432/cn1?sslmode=disable
+          CN1_SQLA_MYSQL: mysql://cn1:cn1secret@127.0.0.1:3306/cn1?sslmode=disable
+
       - name: Run ParparVM Benchmark
         working-directory: vm
         # Each benchmark already creates its own mutator/collector threads. Run
@@ -697,6 +740,70 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
+  python-backend-native:
+    # The Flask and FastAPI samples built NATIVELY (ParparVM, clang) and
+    # driven over sockets by the same client as the JVM run. The only place
+    # Python handlers run on the backend's virtual threads, so the only run
+    # of the GIL's scheduler path. A job of its own: it takes most of an
+    # hour, and vm-tests already fills its limit without it.
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install native build tools
+        run: |
+          bash scripts/ci/apt-get-update.sh
+          sudo apt-get install -y clang zlib1g-dev libcurl4-openssl-dev libssl-dev libnghttp2-dev
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Save JDK 8 Path
+        run: echo "JDK_8_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: Save JDK 17 Path
+        run: echo "JDK_17_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      # Gates off for the reason the vm-tests install step gives.
+      - name: Install core, the backend and the plugin
+        working-directory: maven
+        run: |
+          bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -pl core,backend,backend-test,codenameone-maven-plugin \
+            -am install -DskipTests -Plocal-dev-javase \
+            -Dspotbugs.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true \
+            -Dmaven.javadoc.skip=true \
+            -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo"
+        env:
+          JAVA_HOME: ${{ env.JDK_8_HOME }}
+          JDK_8_HOME: ${{ env.JDK_8_HOME }}
+
+      # python-runtime compiles at release 17, hence JDK 17 for the install.
+      - name: Install the Python transpiler and runtime, and JavaAPI
+        run: |
+          (cd maven && bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -pl python-runtime,python-transpiler \
+            install -DskipTests -Dspotbugs.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true \
+            -Dmaven.javadoc.skip=true -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo")
+          bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -f vm/pom.xml -pl JavaAPI install -DskipTests \
+            -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo"
+        env:
+          JAVA_HOME: ${{ env.JDK_17_HOME }}
+
+      - name: Python backend end to end, native
+        run: bash scripts/python-backend/e2e-test.sh --native "$GITHUB_WORKSPACE/.m2-repo"
+        env:
+          JAVA_HOME: ${{ env.JDK_8_HOME }}
+          JDK_8_HOME: ${{ env.JDK_8_HOME }}
+          JAVA17_HOME: ${{ env.JDK_17_HOME }}
+
   backend-websocket-conformance:
     name: WebSocket conformance (Autobahn)
     runs-on: ubuntu-24.04

.github/workflows/pr.yml

@@ -402,7 +402,15 @@ jobs:
             -Dcn1.binaries="${CN1_BINARIES}" \
             -Pcompile-android \
             -DbackendAnalysis \
-            -pl android,ios,project-model,build-engine,codenameone-maven-plugin,codenameone-gradle-plugin,backend,backend-test,backend-parparvm,javac,android-res-compiler,android-compat,swing-compat,javafx-compat,compose-compat,fxml-compiler,desktop-compat-runtime,cn1-app-kit/backend,cn1-app-kit/common -am verify
+            -pl android,ios,project-model,build-engine,codenameone-maven-plugin,codenameone-gradle-plugin,backend,backend-test,backend-parparvm,javac,android-res-compiler,android-compat,swing-compat,javafx-compat,compose-compat,fxml-compiler,desktop-compat-runtime,cn1-app-kit/backend,cn1-app-kit/common,python-transpiler -am verify
+      # The Python runtime compiles at release 17, so it cannot join the JDK 8
+      # reactor above; it runs against the core that reactor installed. It ships
+      # inside applications, so it is held to the same zero-findings gate.
+      - name: Run SpotBugs for the Python runtime
+        if: ${{ matrix.java-version == 8 }}
+        working-directory: maven
+        run: |
+          JAVA_HOME="${JAVA_HOME_17}" mvn -B -DskipTests=true -Pflutter-modules -pl python-runtime verify
       - name: Type-check the SQLCipher-backed database package
         if: ${{ matrix.java-version == 8 }}
         env:
@@ -620,7 +628,7 @@ jobs:
       - name: Generate static analysis HTML summaries
         if: ${{ !cancelled() && matrix.java-version == 8 }}
         env:
-          QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/compose-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target
+          QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/compose-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target:maven/python-transpiler/target:maven/python-runtime/target
           QUALITY_REPORT_SERVER_URL: ${{ github.server_url }}
           QUALITY_REPORT_REPOSITORY: ${{ github.repository }}
           QUALITY_REPORT_REF: ${{ github.event.pull_request.head.sha || github.sha }}
@@ -777,16 +785,16 @@ jobs:
       - name: Generate quality report summary
         if: ${{ !cancelled() && matrix.java-version == 8 }}
         env:
-          QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/compose-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target
+          QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/compose-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target:maven/python-transpiler/target:maven/python-runtime/target
           # Every project listed here must produce a SpotBugs report and that
           # report must be empty. A missing report fails the gate too, so a
           # SpotBugs run that silently stops running is not mistaken for clean.
-          QUALITY_REPORT_REQUIRED_SPOTBUGS: core-unittests:android:ios:ByteCodeTranslator:codenameone-maven-plugin:build-engine:project-model:codenameone-gradle-plugin:build-hint-catalog:build-hint-tools:backend:backend-test:javac:android-res-compiler:android-compat:swing-compat:javafx-compat:compose-compat:fxml-compiler:desktop-compat-runtime
-          # PMD and Checkstyle run on the core and on the backend runtime, and each
-          # must produce its report: without one, the PMD gate below it
-          # has nothing to read and passes.
-          QUALITY_REPORT_REQUIRED_PMD: core-unittests:backend:backend-test:backend-parparvm
-          QUALITY_REPORT_REQUIRED_CHECKSTYLE: core-unittests:backend:backend-test
+          QUALITY_REPORT_REQUIRED_SPOTBUGS: core-unittests:android:ios:ByteCodeTranslator:codenameone-maven-plugin:build-engine:project-model:codenameone-gradle-plugin:build-hint-catalog:build-hint-tools:backend:backend-test:javac:android-res-compiler:android-compat:swing-compat:javafx-compat:compose-compat:fxml-compiler:desktop-compat-runtime:python-transpiler:python-runtime
+          # PMD and Checkstyle run on the core, the backend runtime and the Python
+          # transpiler and runtime, and each must produce its report: without one,
+          # the PMD gate below it has nothing to read and passes.
+          QUALITY_REPORT_REQUIRED_PMD: core-unittests:backend:backend-test:backend-parparvm:python-transpiler:python-runtime
+          QUALITY_REPORT_REQUIRED_CHECKSTYLE: core-unittests:backend:backend-test:python-transpiler:python-runtime
           QUALITY_REPORT_SERVER_URL: ${{ github.server_url }}
           QUALITY_REPORT_REPOSITORY: ${{ github.repository }}
           QUALITY_REPORT_REF: ${{ github.event.pull_request.head.sha || github.sha }}