docs: Move from sphinx to Mkdocs

.devcontainer/devcontainer.json

@@ -0,0 +1,43 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Python 3",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/python:3.13-bookworm",
+	"features": {
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
+		"ghcr.io/devcontainers/features/git-lfs:1": {},
+		"ghcr.io/va-h/devcontainers-features/uv:1": {
+			"shellautocompletion": true
+		}
+	},
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+	"workspaceFolder": "${localWorkspaceFolder}",
+	"workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
+	"mounts": [
+		"source=${localEnv:HOME}/.cache/pip,target=/home/vscode/.cache/pip,type=bind",
+		"source=${localEnv:HOME}/.cache/uv,target=/home/vscode/.cache/uv,type=bind",
+		"source=${localEnv:HOME}/.cache/huggingface,target=/home/vscode/.cache/huggingface,type=bind",
+	],
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"charliermarsh.ruff"
+			]
+		}
+	}
+}

.gitattributes

@@ -41,9 +41,9 @@
 *.br -text diff
 *.gz -text diff
 *.tar -text diff
-*.zip -text diff
 *.tgz -text diff
 *.tbz2 -text diff
+*.zip binary
 
 # Win Executable
 *.exe binary
@@ -72,4 +72,3 @@
 #docs/_static/img filter=lfs diff=lfs merge=lfs -text
 #services/cogstack-cohort/cui_pt2ch.json filter=lfs diff=lfs merge=lfs -text
 #services/cogstack-cohort/snomed_terms.json filter=lfs diff=lfs merge=lfs -text
-*.zip !text !filter !merge !diff

.github/workflows/doc_build.yml

@@ -21,22 +21,22 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - name: Set up Python
-        uses: actions/setup-python@v7
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: "3.11"
-          cache: "pip"
-          cache-dependency-path: docs/requirements.txt
+          python-version: "3.12"
+          enable-cache: true
+          cache-dependency-glob: "docs/uv.lock"
 
       - name: Run docs QA checks
         run: |
           python3 scripts/tests/lint_markdown.py
           python3 scripts/tests/check_docs_links.py
 
+      - name: Install docs dependencies
+        working-directory: docs
+        run: uv sync --frozen
+
       - name: Build docs
-        run: |
-          cd docs
-          python3 -m pip install --upgrade pip
-          python3 -m pip install -r requirements.txt
-          make clean
-          make html
+        working-directory: docs
+        run: uv run mkdocs build -f ../mkdocs.yml

.gitignore

@@ -44,6 +44,7 @@ security/templates/**
 
 # docs
 docs/build/*
+site/
 
 # Ignore all .env files at any level
 .env

.readthedocs.yaml

@@ -5,13 +5,19 @@
 version: 2
 
 build:
-  os: ubuntu-24.04
+  os: "ubuntu-24.04"
   tools:
-    python: "3.11"
+    python: "3.13"
+  jobs:
+    pre_create_environment:
+        - cd docs
+        - asdf plugin add uv
+        - asdf install uv latest
+        - asdf global uv latest
+    create_environment:
+        - uv venv "${READTHEDOCS_VIRTUALENV_PATH}"
+    install:
+        - cd docs && UV_PROJECT_ENVIRONMENT="${READTHEDOCS_VIRTUALENV_PATH}" uv sync --frozen
 
-sphinx:
-  configuration: docs/conf.py
-
-python:
-  install:
-    - requirements: docs/requirements.txt
+mkdocs:
+  configuration: mkdocs.yml

docs/README.md

@@ -0,0 +1,40 @@
+# CogStack NiFi Documentation
+
+The central documentation for cogstack, hosted on docs.cogstack.org
+
+## Setup
+
+### Prerequisites
+
+- Python 3.10 or higher
+- [uv](https://github.com/astral-sh/uv) package manager
+
+### Installation
+
+```bash
+uv venv --python 3.12 --allow-existing 
+source .venv/bin/activate
+uv sync --dev
+```
+
+## Usage
+
+### Serve documentation locally (Development)
+
+To preview the documentation locally with live reload:
+
+```bash
+uv run mkdocs serve
+```
+
+The documentation will be available at `http://127.0.0.1:8000`
+
+### Build documentation
+
+To build the static site:
+
+```bash
+uv run mkdocs build
+```
+
+The built site will be in the `site/` directory.

docs/index.md

@@ -0,0 +1,93 @@
+# CogStack-NiFi
+
+[![nifi](https://github.com/CogStack/CogStack-NiFi/actions/workflows/docker_nifi.yml/badge.svg?branch=main)](https://github.com/CogStack/CogStack-NiFi/actions/workflows/docker_nifi.yml)
+[![doc-build](https://github.com/CogStack/CogStack-NiFi/actions/workflows/doc_build.yml/badge.svg?branch=main)](https://github.com/CogStack/CogStack-NiFi/actions/workflows/doc_build.yml)
+[![elasticsearch-stack](https://github.com/CogStack/CogStack-NiFi/actions/workflows/docker_elasticsearch_stack.yml/badge.svg?branch=main)](https://github.com/CogStack/CogStack-NiFi/actions/workflows/docker_elasticsearch_stack.yml)
+
+## 💡 Introduction
+
+This repository proposes a possible next step in the evolution of free-text data processing originally implemented in [CogStack-Pipeline](https://github.com/CogStack/CogStack-Pipeline), moving towards a more modular, Platform-as-a-Service (PaaS) approach.
+
+**CogStack-NiFi** demonstrates how to use [Apache NiFi](https://nifi.apache.org/) as the central data workflow engine for clinical document processing, integrating services such as text extraction and natural language processing (NLP). Each component runs as a standalone service, with NiFi handling data routing between components and data sources/sinks.
+
+All NLP/ML/data services are expected to implement a uniform RESTful API, allowing seamless integration into existing pipelines and making it easy to incorporate any NLP application into the stack.
+
+---
+
+## ⚠️ Important Notice
+
+This project is under active development. New features or services may impact existing deployments. Please review the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news.html) and [documentation](https://cogstack-nifi.readthedocs.io) before upgrading.
+
+---
+
+## 💬 Asking Questions
+
+Need help? Feel free to:
+
+- Open an issue on the [GitHub Issue Tracker](https://github.com/CogStack/CogStack-NiFi/issues)
+- Start a discussion on our [Discourse forum](https://discourse.cogstack.org) (actively monitored by the dev team)
+
+---
+
+## 🗂️ Project
+
+This table describes repository layout. For setup and operations, use the deployment and NiFi docs linked below.
+
+| Folder         | Description |
+|----------------|-------------|
+| [`nifi`](https://github.com/CogStack/CogStack-NiFi/tree/main/nifi)         | Custom Apache NiFi Docker image with workflows, configs, drivers, and user resources. |
+| [`security`](https://github.com/CogStack/CogStack-NiFi/tree/main/security) | Scripts for generating SSL certificates and other security-related tools. |
+| [`services`](https://github.com/CogStack/CogStack-NiFi/tree/main/services) | NLP and auxiliary services, each with its own configs and resources. |
+| [`deploy`](https://github.com/CogStack/CogStack-NiFi/tree/main/deploy)     | Example deployment setup, combining NiFi and related services. |
+| [`scripts`](https://github.com/CogStack/CogStack-NiFi/tree/main/scripts)   | Helper scripts (e.g., setup tools, sample DB ingestion, Elasticsearch ingestion). |
+| [`data`](https://github.com/CogStack/CogStack-NiFi/tree/main/data)         | Place any test or data to be ingested here. |
+| [`typings`](https://github.com/CogStack/CogStack-NiFi/tree/main/typings)   | Stubs for code linting/type-hint, etc. |
+
+---
+
+## 📚 Documentation & Getting Started
+
+### Quick Start (5 minutes)
+
+```bash
+# from repository root
+git lfs pull
+make -C deploy git-update-submodules
+make -C deploy help
+make -C deploy start-data-infra
+```
+
+After services start:
+
+- NiFi: `https://localhost:8443`
+- Elasticsearch: `http://localhost:9200`
+- Kibana/OpenSearch Dashboards: `https://localhost:5601`
+
+Stop the core stack with:
+
+```bash
+make -C deploy stop-data-infra
+```
+
+**Prerequisites**:
+
+- Docker + Docker Compose (mandatory)
+- `make`
+- `git` + `git-lfs`
+- `python3.11`
+- Basic Linux/UNIX shell familiarity
+
+📖 Official documentation: [cogstack-nifi.readthedocs.io](https://cogstack-nifi.readthedocs.io/en/latest/)
+
+🚀 New to the project? Start with the [deployment guide](https://cogstack-nifi.readthedocs.io/en/latest/deploy/main.html) for example setups and workflows.
+
+🐞 For troubleshooting or bug reports, consult the [known issues section](https://cogstack-nifi.readthedocs.io/en/latest/deploy/troubleshooting.html) before opening a ticket.
+
+---
+
+## 🛑 Important Updates
+
+Check the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news.html) section regularly for:
+
+- Major changes to project structure or configuration
+- Security advisories or vulnerabilities affecting deployments