feat: localchat, a Go + Templ + HTMX chat app for local Gemma models

Streams replies over SSE from any OpenAI-compatible server (oMLX,
llama.cpp, Ollama, ...). Single binary that can install itself as an
OS service; docker compose bundles llama.cpp + Gemma 4 E2B.

internal/llm/client.go

@@ -0,0 +1,175 @@
+// Package llm is a minimal client for OpenAI-compatible chat APIs, as served
+// by llama.cpp, oMLX, Ollama, LM Studio, Lemonade, vLLM and friends.
+package llm
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// Message is one chat turn.
+type Message struct {
+	Role    string `json:"role"` // "system", "user" or "assistant"
+	Content string `json:"content"`
+}
+
+// Client talks to a single OpenAI-compatible endpoint.
+type Client struct {
+	BaseURL string // e.g. http://127.0.0.1:8000/v1
+	APIKey  string
+	Model   string // empty: use the first model the server lists
+	HTTP    *http.Client
+}
+
+func (c *Client) httpClient() *http.Client {
+	if c.HTTP != nil {
+		return c.HTTP
+	}
+	return http.DefaultClient
+}
+
+func (c *Client) newRequest(ctx context.Context, method, path string, body io.Reader) (*http.Request, error) {
+	req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, body)
+	if err != nil {
+		return nil, err
+	}
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	if c.APIKey != "" {
+		req.Header.Set("Authorization", "Bearer "+c.APIKey)
+	}
+	return req, nil
+}
+
+// Models lists the model ids the server offers. It doubles as a health check.
+func (c *Client) Models(ctx context.Context) ([]string, error) {
+	req, err := c.newRequest(ctx, http.MethodGet, "/models", nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := c.httpClient().Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if err := checkStatus(resp); err != nil {
+		return nil, err
+	}
+	var out struct {
+		Data []struct {
+			ID string `json:"id"`
+		} `json:"data"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
+		return nil, fmt.Errorf("decode models: %w", err)
+	}
+	ids := make([]string, len(out.Data))
+	for i, m := range out.Data {
+		ids[i] = m.ID
+	}
+	return ids, nil
+}
+
+// ResolveModel returns the configured model, or the first one the server
+// lists when none is configured.
+func (c *Client) ResolveModel(ctx context.Context) (string, error) {
+	if c.Model != "" {
+		return c.Model, nil
+	}
+	ids, err := c.Models(ctx)
+	if err != nil {
+		return "", err
+	}
+	if len(ids) == 0 {
+		return "", errors.New("model server lists no models; set LLM_MODEL")
+	}
+	return ids[0], nil
+}
+
+// Stream sends the conversation and calls onDelta for every chunk of the
+// reply as it arrives. It returns when the reply is complete, the context is
+// cancelled, or onDelta returns an error.
+func (c *Client) Stream(ctx context.Context, msgs []Message, onDelta func(string) error) error {
+	model, err := c.ResolveModel(ctx)
+	if err != nil {
+		return err
+	}
+	body, err := json.Marshal(map[string]any{
+		"model":    model,
+		"messages": msgs,
+		"stream":   true,
+	})
+	if err != nil {
+		return err
+	}
+	req, err := c.newRequest(ctx, http.MethodPost, "/chat/completions", bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Accept", "text/event-stream")
+	resp, err := c.httpClient().Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if err := checkStatus(resp); err != nil {
+		return err
+	}
+
+	sc := bufio.NewScanner(resp.Body)
+	sc.Buffer(make([]byte, 64*1024), 1024*1024)
+	for sc.Scan() {
+		data, ok := strings.CutPrefix(sc.Text(), "data:")
+		if !ok {
+			continue // blank separators, comments, "event:" lines
+		}
+		data = strings.TrimSpace(data)
+		if data == "[DONE]" {
+			return nil
+		}
+		var chunk struct {
+			Choices []struct {
+				Delta struct {
+					Content string `json:"content"`
+				} `json:"delta"`
+			} `json:"choices"`
+			Error *struct {
+				Message string `json:"message"`
+			} `json:"error"`
+		}
+		if err := json.Unmarshal([]byte(data), &chunk); err != nil {
+			return fmt.Errorf("decode stream chunk: %w", err)
+		}
+		if chunk.Error != nil {
+			return fmt.Errorf("model server: %s", chunk.Error.Message)
+		}
+		for _, ch := range chunk.Choices {
+			if ch.Delta.Content == "" {
+				continue
+			}
+			if err := onDelta(ch.Delta.Content); err != nil {
+				return err
+			}
+		}
+	}
+	if err := sc.Err(); err != nil {
+		return err
+	}
+	return nil // stream ended without [DONE]; treat what we got as the reply
+}
+
+func checkStatus(resp *http.Response) error {
+	if resp.StatusCode < 300 {
+		return nil
+	}
+	msg, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
+	return fmt.Errorf("model server returned %s: %s", resp.Status, bytes.TrimSpace(msg))
+}

internal/llm/client_test.go

@@ -0,0 +1,107 @@
+package llm
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// fakeServer mimics an OpenAI-compatible server that streams the given chunks.
+func fakeServer(t *testing.T, chunks []string) *httptest.Server {
+	t.Helper()
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v1/models":
+			fmt.Fprint(w, `{"data":[{"id":"tiny-model"},{"id":"other"}]}`)
+		case "/v1/chat/completions":
+			if got := r.Header.Get("Authorization"); got != "Bearer secret" {
+				http.Error(w, `{"error":"bad key"}`, http.StatusUnauthorized)
+				return
+			}
+			var req struct {
+				Model  string    `json:"model"`
+				Stream bool      `json:"stream"`
+				Msgs   []Message `json:"messages"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+				t.Errorf("decode request: %v", err)
+			}
+			if req.Model != "tiny-model" || !req.Stream || len(req.Msgs) != 1 {
+				t.Errorf("unexpected request: %+v", req)
+			}
+			w.Header().Set("Content-Type", "text/event-stream")
+			fmt.Fprint(w, ": keep-alive comment\n\n")
+			fmt.Fprint(w, `data: {"choices":[{"delta":{"role":"assistant"}}]}`+"\n\n")
+			for _, c := range chunks {
+				b, _ := json.Marshal(c)
+				fmt.Fprintf(w, `data: {"choices":[{"delta":{"content":%s}}]}`+"\n\n", b)
+			}
+			fmt.Fprint(w, "data: [DONE]\n\n")
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+}
+
+func TestStream(t *testing.T) {
+	srv := fakeServer(t, []string{"Hel", "lo", "\n", "world"})
+	defer srv.Close()
+
+	c := &Client{BaseURL: srv.URL + "/v1", APIKey: "secret"} // no model: auto-pick first
+	var got strings.Builder
+	err := c.Stream(context.Background(), []Message{{Role: "user", Content: "hi"}}, func(d string) error {
+		got.WriteString(d)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.String() != "Hello\nworld" {
+		t.Fatalf("got %q", got.String())
+	}
+}
+
+func TestStreamHTTPError(t *testing.T) {
+	srv := fakeServer(t, nil)
+	defer srv.Close()
+
+	c := &Client{BaseURL: srv.URL + "/v1", APIKey: "wrong", Model: "tiny-model"}
+	err := c.Stream(context.Background(), []Message{{Role: "user", Content: "hi"}}, func(string) error { return nil })
+	if err == nil || !strings.Contains(err.Error(), "401") {
+		t.Fatalf("want 401 error, got %v", err)
+	}
+}
+
+func TestStreamCallbackErrorStops(t *testing.T) {
+	srv := fakeServer(t, []string{"a", "b", "c"})
+	defer srv.Close()
+
+	c := &Client{BaseURL: srv.URL + "/v1", APIKey: "secret"}
+	stop := fmt.Errorf("stop")
+	n := 0
+	err := c.Stream(context.Background(), []Message{{Role: "user", Content: "hi"}}, func(string) error {
+		n++
+		return stop
+	})
+	if err != stop || n != 1 {
+		t.Fatalf("err=%v n=%d", err, n)
+	}
+}
+
+func TestResolveModel(t *testing.T) {
+	srv := fakeServer(t, nil)
+	defer srv.Close()
+
+	c := &Client{BaseURL: srv.URL + "/v1"}
+	if m, err := c.ResolveModel(context.Background()); err != nil || m != "tiny-model" {
+		t.Fatalf("auto: %q %v", m, err)
+	}
+	c.Model = "pinned"
+	if m, _ := c.ResolveModel(context.Background()); m != "pinned" {
+		t.Fatalf("pinned: %q", m)
+	}
+}