Decided on Parakeet ONNX Runtime. Works pretty great. Realtime voice chat possible now. UX lacking.

stt-parakeet/start_display_server.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Start ASR Display Server with GPU support
+# This script sets up the environment properly for CUDA libraries
+#
+
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+# Activate virtual environment
+if [ -f "venv/bin/activate" ]; then
+    source venv/bin/activate
+else
+    echo "Error: Virtual environment not found at venv/bin/activate"
+    exit 1
+fi
+
+# Get CUDA library paths from venv
+VENV_DIR="$SCRIPT_DIR/venv"
+CUDA_LIB_PATHS=(
+    "$VENV_DIR/lib/python*/site-packages/nvidia/cublas/lib"
+    "$VENV_DIR/lib/python*/site-packages/nvidia/cudnn/lib"
+    "$VENV_DIR/lib/python*/site-packages/nvidia/cufft/lib"
+    "$VENV_DIR/lib/python*/site-packages/nvidia/cuda_nvrtc/lib"
+    "$VENV_DIR/lib/python*/site-packages/nvidia/cuda_runtime/lib"
+)
+
+# Build LD_LIBRARY_PATH
+CUDA_LD_PATH=""
+for pattern in "${CUDA_LIB_PATHS[@]}"; do
+    for path in $pattern; do
+        if [ -d "$path" ]; then
+            if [ -z "$CUDA_LD_PATH" ]; then
+                CUDA_LD_PATH="$path"
+            else
+                CUDA_LD_PATH="$CUDA_LD_PATH:$path"
+            fi
+        fi
+    done
+done
+
+# Export library path
+if [ -n "$CUDA_LD_PATH" ]; then
+    export LD_LIBRARY_PATH="$CUDA_LD_PATH:${LD_LIBRARY_PATH:-}"
+    echo "CUDA libraries path set: $CUDA_LD_PATH"
+else
+    echo "Warning: No CUDA libraries found in venv"
+fi
+
+# Set Python path
+export PYTHONPATH="$SCRIPT_DIR:${PYTHONPATH:-}"
+
+# Run the display server
+echo "Starting ASR Display Server with GPU support..."
+python server/display_server.py "$@"