Generating
The three shapes a generation comes in: text for a person to read, a Go value for code to use, and a stream for a user who is waiting.
Text in, text out
The lowest layer, plus the two things every call site needs and most forget: reading FinishReason (length means the answer was cut off, not that it finished), and telling a temporary failure from a permanent one so only 429 and 5xx are retried.
go
package main
import (
"context"
core "gitlab.finema.co/finema/idin-core/v2"
)
// --- Example 1: text in, text out -------------------------------------------
//
// The lowest layer. core.LLM(ctx) is a function rather than a method on
// IContext for the same reason core.Requester is: calling a model is not a
// capability of the request, it is something you do with the request's deadline
// attached — so a plain context.Context works here too, and a context from
// nowhere gets the disabled model instead of nil.
// summaryRules is a package-level const, not a string built per call, and that
// is the whole point: the system prompt sits at the front of the prompt, so any
// byte that varies in it (a timestamp, the user's name, a request id) moves the
// cache prefix and every request pays full price. Anything that changes belongs
// in a message.
const summaryRules = `Summarise the text in at most three lines.
Answer in the language the text is written in.
State only what the text says — never add facts of your own.`
// summarize is the shape most calls have: one system prompt, one user turn, a
// hard ceiling on the reply.
func summarize(ctx context.Context, body string) (string, core.IError) {
resp, err := core.LLM(ctx).Generate(core.LLMRequest{
System: summaryRules,
CacheSystem: true, // paid in full once, then read at the cached rate
Messages: []core.LLMMessage{core.LLMUser(body)},
// MaxTokens is the only per-call cost ceiling that is actually enforced.
// 0 would fall back to AI_MAX_TOKENS, which is sized for the largest
// caller in the process, not for this one.
MaxTokens: 256,
})
if err != nil {
return "", err
}
return readAnswer(resp)
}
// readAnswer is why FinishReason is not decoration.
//
// "length" means the answer was cut mid-sentence, not that the model finished
// early — and the text still looks like an answer. Returning it is how half a
// summary ends up stored as the real one.
func readAnswer(resp core.LLMResponse) (string, core.IError) {
switch resp.FinishReason {
case core.LLMFinishStop:
return resp.Text, nil
case core.LLMFinishLength:
return "", core.New(502, "AI_RESPONSE_TRUNCATED",
"the model ran out of output tokens — raise MaxTokens or ask for a shorter answer")
case core.LLMFinishContentFilter:
return "", core.New(422, "AI_CONTENT_BLOCKED", "the provider refused to answer this prompt")
case core.LLMFinishToolUse:
// Out of MaxSteps with the model still asking for tools. resp.Text is
// not the final answer — see 04_tools.go.
return "", core.New(500, "AI_DID_NOT_FINISH", "the model stopped mid tool loop")
default:
return "", core.New(502, "AI_UNEXPECTED_FINISH", "the model stopped for an unexpected reason")
}
}
// followUp replays a stored conversation. The framework keeps no history of its
// own: a chat is whatever the service loaded out of its own table, in order,
// oldest first.
//
// LLMAssistant is for replaying what was said, not for prefilling a reply —
// several current models reject a conversation that ends on an assistant turn,
// which is why the new question is appended last.
func followUp(ctx context.Context, history []core.LLMMessage, question string) (string, core.IError) {
msgs := append(append([]core.LLMMessage(nil), history...), core.LLMUser(question))
resp, err := core.LLM(ctx).Generate(core.LLMRequest{
System: summaryRules,
CacheSystem: true,
Messages: msgs,
MaxTokens: 512,
})
if err != nil {
return "", err
}
return readAnswer(resp)
}
// summarizeOrQueue is what "design for failure" looks like at one call site.
//
// The distinction that matters is temporary versus permanent: a 429 or a 5xx is
// the provider asking for time, and the work should end up somewhere that will
// try again with backoff. Everything else returns the same answer however many
// times it is sent, and retrying only pays for it twice.
func summarizeOrQueue(ctx context.Context, docID, body string, queue func(string) core.IError) (string, core.IError) {
text, err := summarize(ctx, body)
switch {
case err == nil:
return text, nil
case err.GetStatus() == 429, err.GetStatus() >= 500:
// Temporary. A job gets timeout, retry, backoff and a run log for free —
// see 05_agent_job.go — where an in-place retry loop gets none of them.
return "", queue(docID)
default:
// Permanent: a malformed request, a rejected prompt, a model that cannot
// see. Retrying is spending money to be told the same thing again.
return "", err
}
}
// costOf reports what one call consumed. CachedInputTokens is reported apart
// from InputTokens because it is billed at a different rate — folding the two
// together makes a cost report wrong in the direction that looks fine.
func costOf(resp core.LLMResponse) (total, cached int) {
return resp.Usage.Total(), resp.Usage.CachedInputTokens
}A value, not a paragraph
llm.New[T] derives the schema from T, constrains the model to it and unmarshals the reply back — with a Validate hook for the rules JSON Schema cannot express. Prompting for "JSON" and parsing it yourself is the same code with one silent failure mode added. See Typed Values.
go
package main
import (
"context"
"time"
core "gitlab.finema.co/finema/idin-core/v2"
"gitlab.finema.co/finema/idin-core/v2/llm"
)
// --- Example 2: a value, not a paragraph ------------------------------------
//
// Generate returns text, which is right for a chat reply and wrong for
// everything else a backend does with a model. Extraction, classification and
// scoring all want a Go value, and getting one out of text means describing a
// schema, asking for JSON, parsing it and checking it — four steps that every
// service would otherwise write again, slightly differently.
//
// llm.New[T] is those four steps: the schema is derived from T by reflection,
// the model is constrained to it, and the reply is unmarshalled back into T.
// The failure mode it removes is the quiet one — a model that helpfully wrote
// "Here is the JSON you asked for:" in front of the object, which json.Unmarshal
// rejects on a line nowhere near the prompt that caused it.
// Invoice is the result type. The jsonschema tag carries what the schema can
// express; anything longer belongs in the system prompt, where wording lives.
//
// A pointer field or omitempty makes a field optional — everything else is
// listed in `required`, and the object is closed, so a model cannot invent an
// extra key and have it silently dropped at unmarshal time.
type Invoice struct {
Vendor string `json:"vendor" jsonschema:"description=Company that issued the invoice"`
Number string `json:"number" jsonschema:"description=Invoice number as printed"`
Total float64 `json:"total" jsonschema:"description=Grand total including tax"`
Status string `json:"status" jsonschema:"enum=draft|sent|paid"`
Due time.Time `json:"due"`
Note string `json:"note,omitempty"`
}
const invoiceRules = `Read the OCR text of a Thai invoice and return its fields.
Amounts are THB. Strip thousands separators.
If a field is genuinely absent, leave it empty rather than guessing.`
// extractInvoice is the common case: text in, value out.
//
// Validate covers the rules a JSON schema cannot express. It runs on the parsed
// value before it is returned, so a caller never has to remember to check —
// which is the difference between a rule and a convention.
func extractInvoice(ctx context.Context, ocr string) (Invoice, core.IError) {
return llm.New[Invoice](ctx).
System(invoiceRules).
// The instructions are long and identical for every document, which is
// exactly the shape prompt caching pays for.
CacheSystem().
// A schema with several fields needs room: hitting the cap truncates the
// JSON, which fails to parse rather than arriving short.
MaxTokens(1024).
Reasoning(core.LLMReasoningLow).
Validate(func(inv Invoice) core.IError {
if inv.Total <= 0 {
return core.New(422, "INVALID_TOTAL", "an invoice with no total was not read correctly")
}
if inv.Due.After(time.Now().AddDate(5, 0, 0)) {
// A due date five years out is the model misreading a Buddhist-era
// year, and it is worth failing on rather than storing.
return core.Newf(422, "INVALID_DUE_DATE", "due date %s is implausible", inv.Due.Format(time.DateOnly))
}
return nil
}).
Extract(ocr)
}
// Category is a one-field result, which is the cheapest useful shape there is:
// a classification that the compiler checks and a switch can branch on.
type Category struct {
Label string `json:"label" jsonschema:"enum=billing|technical|sales|abuse|other"`
Confidence float64 `json:"confidence" jsonschema:"description=0 to 1"`
}
// classifyTicket routes a cheap job to a cheap model.
//
// Model changes the model *within the configured provider*, so the call is
// still metered, logged and attributed like every other. Building a second
// client to change model is the alternative, and its tokens land in nobody's
// dashboard.
func classifyTicket(ctx context.Context, body string) (Category, core.IError) {
return llm.New[Category](ctx).
System("Classify the support ticket. Answer with the label only.").
Model("gemini-flash-lite-latest").
MaxTokens(64).
Extract(body)
}
// extractInvoiceWithCost is the same extraction when the cost has to be
// recorded per document — a per-tenant bill, a spend dashboard, a limit.
//
// Result carries the raw JSON as well, which is the thing to attach to a Sentry
// event when a value comes back wrong: by the time anyone looks, the reply is
// otherwise gone.
func extractInvoiceWithCost(ctx context.Context, ocr string) (llm.Result[Invoice], core.IError) {
// Result, not Generate: the value alone throws away the usage numbers, and
// they are gone for good — the provider does not answer "what did that
// document cost" after the fact.
return llm.New[Invoice](ctx).
System(invoiceRules).
CacheSystem().
MaxTokens(1024).
Ask(ocr).
Result()
}
// reusableExtractor shows why every builder method returns a copy: a
// half-configured value is safe to build once at startup and share, because
// nothing a later call does can mutate it.
func reusableExtractor(ctx context.Context, docs []string) ([]Invoice, core.IError) {
base := llm.New[Invoice](ctx).System(invoiceRules).CacheSystem().MaxTokens(1024)
out := make([]Invoice, 0, len(docs))
for _, doc := range docs {
inv, err := base.Extract(doc) // base is unchanged by this
if err != nil {
return nil, err
}
out = append(out, inv)
}
return out, nil
}Streaming to a browser
An SSE handler, including the parts that bite: deltas need JSON-encoding or a newline breaks the frame, echo v5 hands out a plain http.ResponseWriter so flushing is a type assertion, and Response() is only complete once the loop has ended.
go
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
core "gitlab.finema.co/finema/idin-core/v2"
)
// --- Example 3: streaming to a browser --------------------------------------
//
// A stream is an iterator rather than a channel, on purpose: abandoning a
// channel halfway leaves the producer blocked on a send nobody will receive,
// while Close is an ordinary cleanup call. It is the shape of bufio.Scanner and
// sql.Rows, and it gives the terminal error one place to live — s.Err().
//
// The rule the whole file follows: `defer s.Close()` on the line after the
// error check, always. A user closing the tab is the normal case, not the edge
// case, and it is the one that leaks.
// streamAnswer writes an answer to the client as it arrives, as server-sent
// events.
func streamAnswer(c core.IHTTPContext) error {
s, err := core.LLM(c).Stream(core.LLMRequest{
System: summaryRules,
CacheSystem: true,
Messages: []core.LLMMessage{core.LLMUser(c.QueryParam("q"))},
MaxTokens: 2048,
})
if err != nil {
// Nothing has been written yet, so the framework's error shape still
// reaches the client as JSON with a real status code. Once the first
// byte is out, that is no longer true — see below.
return err
}
defer func() { _ = s.Close() }()
w := c.Response()
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
// Without this nginx buffers the whole response and the user sees nothing
// until it is finished, which is the one thing streaming was for.
w.Header().Set("X-Accel-Buffering", "no")
w.WriteHeader(http.StatusOK)
// echo v5 hands out a plain http.ResponseWriter, so the flusher is a type
// assertion rather than a method. Without flushing, every delta sits in the
// buffer and arrives at once.
flush := func() {
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
}
for s.Next() {
// A delta can contain a newline, which ends an SSE frame early and
// corrupts everything after it. Encoding as JSON is cheaper than
// discovering that from a bug report about "answers that stop halfway".
line, _ := json.Marshal(map[string]string{"t": s.Text()})
if _, werr := fmt.Fprintf(w, "data: %s\n\n", line); werr != nil {
// The client is gone. Returning nil rather than an error keeps this
// out of Sentry: a closed tab is not an incident, and the deferred
// Close releases the provider connection.
return nil
}
flush()
}
if err := s.Err(); err != nil {
// Headers are already sent, so this cannot become a 500. Say so in-band
// and let the client decide what to render.
fmt.Fprintf(w, "event: error\ndata: {\"code\":%q}\n\n", err.GetCode())
flush()
return nil
}
// Everything worth recording is only complete now: usage arrives with the
// last chunk, and a grounded answer's citations arrive as chunks that carry
// no text at all, so they never appeared as a delta.
final := s.Response()
done, _ := json.Marshal(map[string]any{
"finish": string(final.FinishReason),
"tokens": final.Usage.OutputTokens,
})
fmt.Fprintf(w, "event: done\ndata: %s\n\n", done)
flush()
c.Log().Info("streamed an answer",
"finish", string(final.FinishReason),
"input_tokens", final.Usage.InputTokens,
"output_tokens", final.Usage.OutputTokens,
"cached_input_tokens", final.Usage.CachedInputTokens)
return nil
}
// streamAndKeep streams to the user *and* stores the finished answer.
//
// The catch is that s.Response().Text is only complete once Next has returned
// false — mid-stream it holds whatever has arrived so far. Storing it inside
// the loop stores a fragment, and the row looks plausible enough that nobody
// notices.
func streamAndKeep(c core.IHTTPContext, save func(string) core.IError) error {
s, err := core.LLM(c).Stream(core.LLMRequest{
Messages: []core.LLMMessage{core.LLMUser(c.QueryParam("q"))},
MaxTokens: 2048,
})
if err != nil {
return err
}
defer func() { _ = s.Close() }()
w := c.Response()
for s.Next() {
if _, werr := w.Write([]byte(s.Text())); werr != nil {
break // client gone; the partial answer below is still worth keeping
}
}
if err := s.Err(); err != nil {
return err
}
return save(s.Response().Text)
}
// streamThenFinishInBackground is for a generation that must complete even
// though the reader left — a transcript that has to be stored whatever happens.
//
// core.LLM(ctx) is bound to the request, so a disconnect normally cancels the
// generation, which is what you want. WithoutCancel keeps everything the
// context carries (the App above all) while cutting the cancellation wire, and
// the explicit timeout is what stops it from becoming an unbounded goroutine.
func streamThenFinishInBackground(ctx context.Context, question string, save func(string) core.IError) core.IError {
bg, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Minute)
defer cancel()
s, err := core.LLM(ctx).WithContext(bg).Stream(core.LLMRequest{
Messages: []core.LLMMessage{core.LLMUser(question)},
MaxTokens: 2048,
})
if err != nil {
return err
}
defer func() { _ = s.Close() }()
for s.Next() { //nolint:revive // draining is the point; the deltas go nowhere
}
if err := s.Err(); err != nil {
return err
}
return save(s.Response().Text)
}