Getting bytes in and out
The two halves of the job, and the decisions inside each that are much easier to make once than to fix later.
Uploading
The key is generated, the content type is decided by us, and nothing is buffered — three things a hand-rolled upload handler usually gets wrong. Storage also fails loudly where the cache degrades: a dropped upload is a file nobody can get back. See Uploading.
go
package main
import (
"fmt"
"io"
"net/http"
"path"
core "gitlab.finema.co/finema/idin-core/v2"
"gitlab.finema.co/finema/idin-core/v2/errmsgs"
"gitlab.finema.co/finema/idin-core/v2/utils"
)
// --- Example 1: putting bytes in a bucket -----------------------------------
//
// ctx.Storage() returns a handle already bound to the request, so the methods
// take no ctx and a client that hangs up aborts the upload it started.
//
// Storage does not degrade the way the cache does. A service with no S3_*
// configuration still gets a working handle, but every call on it fails with
// STORAGE_DISABLED — because a cache miss is recoverable (recompute the value)
// while an upload that was quietly dropped is a file the caller believes it
// saved and nobody can get back.
// maxUpload is what the service accepts through itself. Anything genuinely large
// should skip the process entirely — see 03_presign.go.
const maxUpload = 10 << 20 // 10 MiB
// documentKey builds the whole address of an object. Three decisions live here,
// and all three are easier to make once than to fix later:
//
// - the prefix is structured (tenant → entity), so a tenant can be listed,
// deleted or migrated without guessing which keys are theirs;
// - the name is a UUID, because a filename the user supplied is untrusted and
// because two people uploading "cv.pdf" must not become one object;
// - the extension is kept, so a content type can still be guessed and a link
// still looks like a file.
//
// The readable name is not lost — it goes in the database row and in
// Content-Disposition. The key must be unguessable and the filename must be
// readable, and those are different jobs.
func documentKey(tenantID, entity, filename string) string {
return path.Join("tenants", tenantID, entity, utils.NewUUID()+path.Ext(filename))
}
// uploadReceipt stores something the service generated itself, so the type and
// the download name are known facts rather than claims from a client.
func uploadReceipt(ctx core.IContext, tenantID string, pdf []byte) (string, core.IError) {
key := documentKey(tenantID, "receipts", "receipt.pdf")
if err := ctx.Storage().PutBytes(key, pdf, core.StoragePutOptions{
ContentType: "application/pdf",
// non-ASCII survives: the header is written in both the quoted and the
// RFC 5987 form
Attachment: "ใบเสร็จ.pdf",
// x-amz-meta-*, handed back by Stat. Small facts that should travel with
// the object — not a database: it cannot be queried and is only visible
// one object at a time
Metadata: map[string]string{"tenant": tenantID},
}); err != nil {
return "", err
}
return key, nil
}
// uploadFromRequest streams a browser upload straight through to the bucket.
// Nothing is buffered, so a 200MB file does not become 200MB of heap.
func uploadFromRequest(c core.IHTTPContext) error {
file, err := c.FormFile("file")
if err != nil {
return c.NewError(err, errmsgs.BadRequest)
}
// file.Size is the real size from the multipart parser — but the whole body
// has already arrived by the time we can read it. To reject earlier, bound
// the request body in middleware; to not receive it at all, presign.
if file.Size > maxUpload {
return c.NewError(nil, errmsgs.BadRequest)
}
src, err := file.Open()
if err != nil {
return c.NewError(err, errmsgs.BadRequest)
}
defer func() { _ = src.Close() }()
key := documentKey(tenantOf(c), "uploads", file.Filename)
// The Content-Type header is the uploader's claim, and an HTML file stored as
// text/html *renders* when it is served — a stored-XSS hole. For anything a
// user supplied, force a download instead of trusting the label.
if err := c.Storage().Put(key, src, core.StoragePutOptions{
ContentType: "application/octet-stream",
Attachment: file.Filename,
}); err != nil {
return err
}
// hand back the key, so the client can reference the object without the
// service having to invent a URL for it
return c.JSON(http.StatusOK, map[string]any{"key": key, "filename": file.Filename})
}
// uploadGenerated writes a file that never exists on disk and never exists whole
// in memory: the generator writes into one end of a pipe while the uploader
// reads the other.
func uploadGenerated(ctx core.IContext, key string, rows []string) core.IError {
pr, pw := io.Pipe()
go func() {
// CloseWithError is the part that matters. Without it, a generator that
// fails half way closes the pipe cleanly and produces a truncated object
// that uploads *successfully*.
_ = pw.CloseWithError(writeRows(pw, rows))
}()
return ctx.Storage().Put(key, pr, core.StoragePutOptions{
ContentType: "text/csv",
Attachment: "orders.csv",
})
}
func writeRows(w io.Writer, rows []string) error {
for _, row := range rows {
if _, err := fmt.Fprintln(w, row); err != nil {
return err
}
}
return nil
}
// tenantOf reads the tenant off the authenticated user. ContextUser carries the
// identity the auth middleware resolved; anything beyond the named fields lives
// in Data.
func tenantOf(c core.IHTTPContext) string {
if user := c.GetUser(); user != nil {
return user.Data["tenant_id"]
}
return "public"
}Reading, listing, deleting
Streaming an object through the service, and why you usually should not. The absent key is an ordinary 404, List is a paginated scan rather than an index, and Move is a copy plus a delete because S3 has no rename.
go
package main
import (
"errors"
"net/http"
"strconv"
core "gitlab.finema.co/finema/idin-core/v2"
"gitlab.finema.co/finema/idin-core/v2/errmsgs"
)
// --- Example 2: reading, listing, deleting ----------------------------------
//
// Object storage is not a filesystem, and most storage bugs come from treating
// it like one: the key space is flat, an object is immutable, there is no
// rename, and "ls" is a paginated scan. Reading is a network call with network
// latency and network failures — including the most ordinary one of all, the key
// that is not there.
// readSmallObject reads a whole object into memory. That is exactly Get plus
// io.ReadAll, so the size limit is however much memory you are willing to spend
// per concurrent caller — fine for a JSON blob, wrong for a video.
func readSmallObject(ctx core.IContext, key string) ([]byte, core.IError) {
data, err := ctx.Storage().GetBytes(key)
if errors.Is(err, core.ErrObjectNotFound) {
// an absent key is an answer, not a failure: it becomes a 404, and it
// must not be reported to Sentry as a server error
return nil, ctx.NewError(err, errmsgs.NotFound)
}
if err != nil {
return nil, err
}
return data, nil
}
// serveObject proxies an object through the service. Do this when access has to
// be *checked* — a private document, a per-user file. Otherwise prefer a signed
// link (03_presign.go): proxying moves every byte through the process, and one
// large download holds a request slot for its whole duration.
func serveObject(c core.IHTTPContext, key string) error {
// Stat first, because the response needs the length and the type — and it
// transfers no body, so it costs one cheap round trip rather than a guess
info, err := c.Storage().Stat(key)
if errors.Is(err, core.ErrObjectNotFound) {
return c.NewError(err, errmsgs.NotFound)
}
if err != nil {
return err
}
body, err := c.Storage().Get(key)
if err != nil {
return err
}
// Close it. An unclosed body holds an HTTP connection from the SDK's pool, so
// a handler that leaks one per request runs out of connections rather than
// out of memory — a far more confusing outage.
defer func() { _ = body.Close() }()
c.Response().Header().Set("Content-Length", strconv.FormatInt(info.Size, 10))
return c.Stream(http.StatusOK, info.ContentType, body)
}
// listTenantObjects lists one tenant's objects through a prefixed handle, so no
// call site ever writes the tenant id into a key and no call site can get it
// wrong.
//
// A Limit is passed on purpose: without one, List follows pagination to the end
// and returns every object under the prefix — many round trips and a large slice
// for a prefix nobody has bounded.
func listTenantObjects(ctx core.IContext, tenantID, prefix string) ([]core.StorageObject, core.IError) {
tenant := ctx.Storage().WithPrefix("tenants/" + tenantID)
// keys come back without the storage prefix — the same names they were
// written under, so a listing feeds straight back into Get or Delete
return tenant.List(prefix, core.StorageListOptions{Limit: 100})
}
// checkThenRead is the mistake worth naming: two round trips to answer one
// question, and the object can disappear between them. Just Get and handle the
// not-found. Exists earns its keep when the answer itself is the product ("is
// this upload finished yet"), not as a guard.
func checkThenRead(ctx core.IContext, key string) ([]byte, core.IError) {
ok, err := ctx.Storage().Exists(key) // absence is false, not an error
if err != nil || !ok {
return nil, err
}
return ctx.Storage().GetBytes(key)
}
// promoteUpload moves an object from its temporary key to its final one. S3 has
// no rename, so Move is a copy followed by a delete and therefore not atomic: a
// failure in between leaves both keys. For this pattern that is harmless — the
// temporary key is swept by a lifecycle rule.
//
// Copy happens inside S3: the bytes never travel to this process, so promoting a
// 2GB object costs one API call rather than 2GB in each direction.
func promoteUpload(ctx core.IContext, tmpKey, finalKey string) core.IError {
return ctx.Storage().Move(tmpKey, finalKey)
}
// deleteDocument removes the object after the row that referenced it is gone.
// Delete takes any number of keys, batches them into calls of 1000, and does not
// mind an absent one — which makes cleanup code idempotent for free.
//
// Unless the bucket has versioning, a delete is permanent. For anything a user
// can trigger, prefer marking the row deleted and letting a lifecycle rule
// remove the object after a grace period.
func deleteDocument(ctx core.IContext, keys ...string) core.IError {
return ctx.Storage().Delete(keys...)
}