Improvements

.dockerignore

@@ -0,0 +1,22 @@
+# The NiFi image only copies pyproject.toml and the nifi source tree.
+# Keep the build context small and prevent unrelated repository content,
+# credentials, and local development state from entering image layers.
+**
+!pyproject.toml
+!uv.lock
+!nifi/
+!nifi/**
+
+# Local NiFi runtime state. These files may contain environment-specific
+# endpoints, identifiers, or other data and are not reproducible inputs.
+nifi/conf/flow.json.gz
+nifi/conf/archive/
+
+# Development and operating-system artefacts within the included source tree.
+nifi/**/__pycache__/
+nifi/**/*.py[cod]
+nifi/**/.DS_Store
+nifi/**/.pytest_cache/
+nifi/**/.ruff_cache/
+nifi/**/.mypy_cache/
+nifi/**/*.egg-info/

.github/dependabot.yml

@@ -14,9 +14,15 @@ updates:
     registries:
       - dockerhub
 
+  # Keep the root Python project and lock file up to date.
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
 registries:
   dockerhub:
     type: "docker-registry"
     url: "https://index.docker.io"
     username: "${{secrets.DOCKER_HUB_USERNAME}}"
-    password: "${{secrets.DOCKER_HUB_ACCESS_TOKEN}}"
+    password: "${{secrets.DOCKER_HUB_ACCESS_TOKEN}}"

.github/workflows/_docker_template.yml

@@ -125,20 +125,40 @@ jobs:
             type=ref,event=branch,enable=${{ github.event_name != 'pull_request' }}
             type=sha,format=short
 
-      - name: Wait for Docker Hub to register images
-        run: sleep 60
-
       - name: Create multi-arch manifest
         shell: bash
         run: |
           set -euo pipefail
+
+          wait_for_image() {
+            local image_ref="$1"
+            local attempt
+
+            for attempt in {1..12}; do
+              if docker buildx imagetools inspect "$image_ref" >/dev/null 2>&1; then
+                echo "Available: ${image_ref}"
+                return 0
+              fi
+
+              if [[ "$attempt" -lt 12 ]]; then
+                echo "Waiting for ${image_ref} (${attempt}/12)..."
+                sleep 5
+              fi
+            done
+
+            echo "Timed out waiting for ${image_ref}" >&2
+            return 1
+          }
+
           while IFS= read -r ref; do
             [[ -z "$ref" ]] && continue
             img="${ref%%:*}"
             tag="${ref#*:}"
+            wait_for_image "${img}:${tag}-amd64"
+            wait_for_image "${img}:${tag}-arm64"
             echo "🌀 Creating manifest for ${img}:${tag}"
             docker buildx imagetools create \
               --tag "${img}:${tag}" \
               "${img}:${tag}-amd64" \
               "${img}:${tag}-arm64"
-          done < <(printf "%s" "${{ steps.meta_manifest.outputs.tags }}")
+          done < <(printf "%s\n" "${{ steps.meta_manifest.outputs.tags }}")

.github/workflows/compose_config.yml

@@ -0,0 +1,37 @@
+name: compose-config
+
+on:
+  push:
+    branches: ['**']
+    paths:
+      - "deploy/*.env"
+      - "deploy/export_env_vars.sh"
+      - "deploy/services*.yml"
+      - "security/env/*.env"
+      - ".github/workflows/compose_config.yml"
+  pull_request:
+    paths:
+      - "deploy/*.env"
+      - "deploy/export_env_vars.sh"
+      - "deploy/services*.yml"
+      - "security/env/*.env"
+      - ".github/workflows/compose_config.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: Validate Docker Compose configuration
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Validate Compose files
+        shell: bash
+        run: |
+          set -euo pipefail
+          source deploy/export_env_vars.sh
+          docker compose -f deploy/services.yml config --quiet
+          docker compose -f deploy/services.dev.yml config --quiet

.github/workflows/dependency_hygiene.yml

@@ -4,11 +4,23 @@ on:
   workflow_dispatch:
   schedule:
     - cron: "0 6 * * 1" # weekly Monday UTC
+  push:
+    branches: ['**']
+    paths:
+      - "docs/pyproject.toml"
+      - "docs/uv.lock"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "scripts/maintenance/audit_pip_dependency.sh"
+      - ".github/workflows/dependency_hygiene.yml"
   pull_request:
     paths:
-      - "docs/requirements.txt"
-      - "nifi/requirements.txt"
+      - "docs/pyproject.toml"
+      - "docs/uv.lock"
+      - "pyproject.toml"
+      - "uv.lock"
       - "scripts/maintenance/audit_pip_dependency.sh"
+      - ".github/workflows/dependency_hygiene.yml"
 
 permissions:
   contents: read
@@ -23,7 +35,7 @@ jobs:
         uses: actions/cache@v6
         with:
           path: ~/.cache/pip
-          key: pip-${{ runner.os }}-${{ hashFiles('docs/requirements.txt', 'nifi/requirements.txt', 'scripts/maintenance/audit_pip_dependency.sh') }}
+          key: pip-${{ runner.os }}-${{ hashFiles('docs/pyproject.toml', 'docs/uv.lock', 'pyproject.toml', 'uv.lock', 'scripts/maintenance/audit_pip_dependency.sh') }}
           restore-keys: |
             pip-${{ runner.os }}-
 
@@ -32,7 +44,12 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install pip-audit
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version-file: "pyproject.toml"
+
+      - name: Install audit tools
         run: python -m pip install --upgrade pip pip-audit
 
       - name: Ensure script is executable

.github/workflows/doc_build.yml

@@ -2,9 +2,8 @@ name: doc-build
 
 on:
   push:
-    branches: [ main ]
+    branches: ['**']
   pull_request:
-    branches: [ main ]
 
   release:
     types: [published]
@@ -24,6 +23,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
+          version-file: "pyproject.toml"
           python-version: "3.12"
           enable-cache: true
           cache-dependency-glob: "docs/uv.lock"
@@ -39,4 +39,4 @@ jobs:
 
       - name: Build docs
         working-directory: docs
-        run: uv run mkdocs build -f ../mkdocs.yml
+        run: uv run mkdocs build --strict -f ../mkdocs.yml

.github/workflows/python_quality.yml

@@ -5,17 +5,15 @@ on:
     branches: ['**']
     paths:
       - "pyproject.toml"
-      - "nifi/requirements.txt"
-      - "nifi/requirements-dev.txt"
+      - "uv.lock"
       - "nifi/**/*.py"
       - "scripts/**/*.py"
       - "typings/**/*.pyi"
       - ".github/workflows/python_quality.yml"
   pull_request:
     paths:
       - "pyproject.toml"
-      - "nifi/requirements.txt"
-      - "nifi/requirements-dev.txt"
+      - "uv.lock"
       - "nifi/**/*.py"
       - "scripts/**/*.py"
       - "typings/**/*.pyi"
@@ -39,17 +37,18 @@ jobs:
           python-version: "3.11"
 
       - name: Install uv
-        run: |
-          python -m pip install --upgrade pip uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version-file: "pyproject.toml"
 
-      - name: Install requirements with uv
+      - name: Install locked project dependencies
         run: |
-          uv pip install --system -r nifi/requirements.txt -r nifi/requirements-dev.txt
+          uv sync --locked --dev
 
       - name: Ruff (critical rules)
         run: |
-          python -m ruff check scripts nifi/user_python_extensions --select F,E9
+          uv run --locked ruff check scripts nifi/user_python_extensions --select F,E9
 
       - name: MyPy
         run: |
-          python -m mypy --config-file pyproject.toml nifi/user_python_extensions scripts
+          uv run --locked mypy --config-file pyproject.toml nifi/user_python_extensions scripts

.github/workflows/shell_script_enforce_exec_check.yml

@@ -40,3 +40,39 @@ jobs:
             echo "Fix with: git update-index --chmod=+x <file>"
             exit 1
           fi
+
+  shell-quality:
+    name: ShellCheck and syntax
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Check root-owned shell scripts
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          scripts=()
+          while IFS= read -r -d '' file; do
+            scripts+=("$file")
+          done < <(
+            git ls-files -z -- '*.sh' \
+              ':(exclude)services/**' \
+              ':(exclude)deploy/charts/**'
+          )
+
+          if [[ "${#scripts[@]}" -eq 0 ]]; then
+            echo "No root-owned shell scripts found."
+            exit 0
+          fi
+
+          echo "Checking ${#scripts[@]} root-owned shell scripts."
+          docker run --rm \
+            --volume "${GITHUB_WORKSPACE}:/mnt:ro" \
+            --workdir /mnt \
+            koalaman/shellcheck:v0.11.0@sha256:61862eba1fcf09a484ebcc6feea46f1782532571a34ed51fedf90dd25f925a8d \
+            "${scripts[@]}"
+
+          for file in "${scripts[@]}"; do
+            bash -n "$file"
+          done

.github/workflows/smoke_gitea_service.yml

@@ -24,10 +24,11 @@ jobs:
           java-version: '21'
 
       - name: Generate certificates
-        working-directory: security/scripts
         run: |
           set -euo pipefail
-          ./create_root_ca_cert.sh
+          make -C deploy init-security-gitea
+          sudo chown 1000:1000 security/certificates/gitea/gitea.key
+          sudo chmod 600 security/certificates/gitea/gitea.key
 
       - name: Start Gitea service
         run: |

.readthedocs.yaml

@@ -10,14 +10,12 @@ build:
     python: "3.13"
   jobs:
     pre_create_environment:
-        - cd docs
         - asdf plugin add uv
-        - asdf install uv latest
-        - asdf global uv latest
+        - UV_VERSION="$(sed -n 's/^required-version = "==\([^"]*\)"$/\1/p' pyproject.toml)" && test -n "${UV_VERSION}" && asdf install uv "${UV_VERSION}" && asdf global uv "${UV_VERSION}"
     create_environment:
         - uv venv "${READTHEDOCS_VIRTUALENV_PATH}"
     install:
         - cd docs && UV_PROJECT_ENVIRONMENT="${READTHEDOCS_VIRTUALENV_PATH}" uv sync --frozen
 
 mkdocs:
-  configuration: mkdocs.yml
+  configuration: mkdocs.yml

README.md

@@ -16,7 +16,7 @@ All NLP/ML/data services are expected to implement a uniform RESTful API, allowi
 
 ## ⚠️ Important Notice
 
-This project is under active development. New features or services may impact existing deployments. Please review the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news.html) and [documentation](https://cogstack-nifi.readthedocs.io) before upgrading.
+This project is under active development. New features or services may impact existing deployments. Please review the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news/) and [documentation](https://cogstack-nifi.readthedocs.io) before upgrading.
 
 ---
 
@@ -54,6 +54,7 @@ This table describes repository layout. For setup and operations, use the deploy
 git lfs pull
 make -C deploy git-update-submodules
 make -C deploy help
+make -C deploy init-security
 make -C deploy start-data-infra
 ```
 
@@ -79,15 +80,15 @@ make -C deploy stop-data-infra
 
 📖 Official documentation: [cogstack-nifi.readthedocs.io](https://cogstack-nifi.readthedocs.io/en/latest/)
 
-🚀 New to the project? Start with the [deployment guide](https://cogstack-nifi.readthedocs.io/en/latest/deploy/main.html) for example setups and workflows.
+🚀 New to the project? Start with the [deployment guide](https://cogstack-nifi.readthedocs.io/en/latest/deploy/main/) for example setups and workflows.
 
-🐞 For troubleshooting or bug reports, consult the [known issues section](https://cogstack-nifi.readthedocs.io/en/latest/deploy/troubleshooting.html) before opening a ticket.
+🐞 For troubleshooting or bug reports, consult the [known issues section](https://cogstack-nifi.readthedocs.io/en/latest/deploy/troubleshooting/) before opening a ticket.
 
 ---
 
 ## 🛑 Important Updates
 
-Check the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news.html) section regularly for:
+Check the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news/) section regularly for:
 
 - Major changes to project structure or configuration
 - Security advisories or vulnerabilities affecting deployments