feat: localchat, a Go + Templ + HTMX chat app for local Gemma models
Streams replies over SSE from any OpenAI-compatible server (oMLX, llama.cpp, Ollama, ...). Single binary that can install itself as an OS service; docker compose bundles llama.cpp + Gemma 4 E2B.
This commit is contained in:
28 files changed
+2459
No files matched your search
@@ -0,0 +1,44 @@
|
||||
# localchat + a local Gemma 4 E2B model served by llama.cpp.
|
||||
#
|
||||
# docker compose up -d # first start downloads the model (~3 GB)
|
||||
# open http://localhost:3000
|
||||
#
|
||||
# Swap the model with LLM_HF_REPO, e.g. LLM_HF_REPO=unsloth/gemma-4-E4B-it-GGUF:Q4_K_M
|
||||
services:
|
||||
llm:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server
|
||||
command:
|
||||
- -hf
|
||||
- ${LLM_HF_REPO:-unsloth/gemma-4-E2B-it-GGUF:Q4_K_M}
|
||||
- --no-mmproj # text-only chat; skip the vision projector download
|
||||
- --ctx-size
|
||||
- "8192"
|
||||
- --port
|
||||
- "8080"
|
||||
environment:
|
||||
LLAMA_CACHE: /models
|
||||
volumes:
|
||||
- models:/models
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fs", "http://localhost:8080/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
start_period: 15m # first boot downloads the model
|
||||
restart: unless-stopped
|
||||
|
||||
app:
|
||||
build: .
|
||||
image: localchat:latest
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
LLM_BASE_URL: http://llm:8080/v1
|
||||
LOCALCHAT_TITLE: ${LOCALCHAT_TITLE:-localchat}
|
||||
SYSTEM_PROMPT: ${SYSTEM_PROMPT:-}
|
||||
depends_on:
|
||||
llm:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
models:
|
||||
Reference in new issue
Block a user