core: route the fatal exit(1) sites through ace_fatal(), behind ACESTEP_FATAL_THROWS

CMakeLists.txt

@@ -121,6 +121,19 @@ add_library(acestep-core STATIC
 target_link_libraries(acestep-core PUBLIC yyjson)
 link_ggml_backends(acestep-core)
 
+# ace_fatal() exit(1)s by default; with this ON it throws ace_fatal_error so an
+# embedder can catch a failed load instead of dying (see src/ace-fatal.h). It
+# gates the body of an inline function, so it is PUBLIC and build-wide -- the one
+# correct way to turn it on, uniformly across acestep-core and its consumers, the
+# way GGML_MAX_NAME is. Defining it on only some targets is an ODR violation.
+# OFF by default, and do not turn it on until the inference path is
+# exception-safe: today only the model-store load path frees cleanly on a throw
+# (a forward-time fatal still leaks -- see the ACESTEP_FATAL_THROWS follow-up).
+option(ACESTEP_FATAL_THROWS "ace_fatal() throws instead of exit(1) (load-path only so far)" OFF)
+if(ACESTEP_FATAL_THROWS)
+    target_compile_definitions(acestep-core PUBLIC ACESTEP_FATAL_THROWS)
+endif()
+
 # ace-synth: full pipeline (text-enc + cond + dit + vae + wav)
 add_executable(ace-synth tools/ace-synth.cpp)
 target_link_libraries(ace-synth PRIVATE acestep-core)

src/ace-fatal.h

@@ -0,0 +1,92 @@
+#pragma once
+// One place the engine says "this load cannot continue."
+//
+// By default ace_fatal() prints the message to stderr and exit(1)s -- byte for
+// byte what the ~25 `fprintf(stderr, "[X] FATAL: ...\n", ...); exit(1);` sites
+// did inline before. That is correct for the CLIs.
+//
+// Define ACESTEP_FATAL_THROWS (the app does; the CLIs do not) and the same
+// message is thrown as ace_fatal_error instead, so C++ stack unwinding runs:
+// ModelStore's LoadGuard frees the half-built module (its backend, scheduler,
+// ggml_context and weight buffer) and its std::lock_guard unlocks, rather than
+// the process dying mid-load. The message is echoed to stderr *and* carried on
+// the exception. (This covers the load path; an inference-time fatal still
+// unwinds through code that is not yet exception-safe -- see the follow-up.)
+//
+// Where the exception ends up depends on who raised it. ModelStore catches
+// ace_fatal_error at the load boundary (store_require_*) and turns a failed load
+// into the nullptr it already returned on failure -- so load callers are
+// unchanged and read the reason from stderr. A fatal raised anywhere without
+// such a boundary propagates to the caller carrying the message.
+//
+// ace_fatal_error is defined even in the default (non-throwing) build so callers
+// can `catch (const ace_fatal_error &)` unconditionally; it is simply never
+// thrown there. Catching this specific type rather than `...` lets an unrelated
+// std::bad_alloc keep propagating instead of being masked as a benign failure.
+//
+// ACESTEP_FATAL_THROWS gates the *body* of an inline function, so it must be
+// defined uniformly for the whole build -- a target-wide compile definition, the
+// way GGML_MAX_NAME already is. Defining it for only some translation units in a
+// binary is an ODR violation: the two bodies differ, the linker silently keeps
+// one, and fatals in the odd TUs would exit(1) instead of throwing.
+//
+// Why an exception and not the alternatives (see #17):
+//   - setjmp/longjmp skips C++ destructors -- it would leak the half-built
+//     module's ggml_context and backend buffer and leave the lock_guard held,
+//     deadlocking the next call.
+//   - Propagating a bool up the chain is a ~2000-line diff across seven headers
+//     for the ~1000 straight-line load calls, and the right behaviour on
+//     "tensor missing" is to abandon the whole load anyway.
+#include <cstdarg>
+#include <cstdio>
+#include <cstdlib>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+#if defined(__GNUC__) || defined(__clang__)
+#    define ACE_FATAL_PRINTF_FMT __attribute__((format(printf, 2, 3)))
+#else
+#    define ACE_FATAL_PRINTF_FMT
+#endif
+
+// Carries the exit code the CLI would have used and the formatted message.
+// Defined unconditionally; only thrown under ACESTEP_FATAL_THROWS.
+struct ace_fatal_error : std::runtime_error {
+    int code;
+    ace_fatal_error(int c, std::string msg) : std::runtime_error(std::move(msg)), code(c) {}
+};
+
+[[noreturn]] ACE_FATAL_PRINTF_FMT inline void ace_fatal(int code, const char * fmt, ...) {
+    va_list ap;
+    va_start(ap, fmt);
+#ifdef ACESTEP_FATAL_THROWS
+    // Format into a string so the message rides with the exception, and echo it
+    // to stderr so the diagnostic is present whether or not a caller catches.
+    va_list ap2;
+    va_copy(ap2, ap);
+    char        stackbuf[1024];
+    int         n = vsnprintf(stackbuf, sizeof(stackbuf), fmt, ap);
+    std::string msg;
+    if (n < 0) {
+        // Encoding error mid-format: the expanded text is unavailable. Mark it as
+        // such rather than pass the raw format string (unexpanded %s and all) off
+        // as if it were the diagnostic.
+        msg = std::string("ace_fatal: could not format message: ") + fmt;
+    } else if (static_cast<size_t>(n) < sizeof(stackbuf)) {
+        msg.assign(stackbuf, static_cast<size_t>(n));
+    } else {
+        msg.resize(static_cast<size_t>(n) + 1);
+        vsnprintf(&msg[0], msg.size(), fmt, ap2);
+        msg.resize(static_cast<size_t>(n));  // drop the NUL vsnprintf wrote
+    }
+    va_end(ap2);
+    va_end(ap);
+    fputs(msg.c_str(), stderr);
+    throw ace_fatal_error(code, std::move(msg));
+#else
+    vfprintf(stderr, fmt, ap);
+    va_end(ap);
+    exit(code);
+#endif
+}

src/backend.h

@@ -5,11 +5,13 @@
 // keep CPU as fallback. This avoids duplicating init logic across
 // qwen3.h, qwen3-lm.h, cond.h, dit.h, vae.h.
 
+#include "ace-fatal.h"
 #include "ggml-backend.h"
 
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <string>
 #include <thread>
 
 struct BackendPair {
@@ -109,19 +111,19 @@ static BackendPair backend_init(const char * label) {
     if (force_backend) {
         bp.backend = ggml_backend_init_by_name(force_backend, nullptr);
         if (!bp.backend) {
-            fprintf(stderr, "[Load] FATAL: GGML_BACKEND=%s not found. Available:", force_backend);
+            std::string avail;
             for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
-                fprintf(stderr, " %s", ggml_backend_dev_name(ggml_backend_dev_get(i)));
+                avail += ' ';
+                avail += ggml_backend_dev_name(ggml_backend_dev_get(i));
             }
-            fprintf(stderr, "\n");
-            exit(1);
+            ace_fatal(1, "[Load] FATAL: GGML_BACKEND=%s not found. Available:%s\n", force_backend,
+                      avail.c_str());
         }
     } else {
         bp.backend = ggml_backend_init_best();
     }
     if (!bp.backend) {
-        fprintf(stderr, "[Load] FATAL: no backend available\n");
-        exit(1);
+        ace_fatal(1, "[Load] FATAL: no backend available\n");
     }
     bool best_is_cpu = (strcmp(ggml_backend_name(bp.backend), "CPU") == 0);
     int  n_threads   = backend_cpu_n_threads();
@@ -133,8 +135,14 @@ static BackendPair backend_init(const char * label) {
         bp.cpu_backend = cpu_backend_new(n_threads);
     }
     if (!bp.cpu_backend) {
-        fprintf(stderr, "[Load] FATAL: failed to init CPU backend\n");
-        exit(1);
+        // Under ACESTEP_FATAL_THROWS this throws, and bp is a local that never
+        // reaches a module or the backend cache -- so free the GPU backend
+        // allocated above (the best_is_cpu branch already freed it and left
+        // bp.backend null) before unwinding, or it leaks. exit(1) does not care.
+        if (bp.backend) {
+            ggml_backend_free(bp.backend);
+        }
+        ace_fatal(1, "[Load] FATAL: failed to init CPU backend\n");
     }
     bp.has_gpu = !best_is_cpu;
     fprintf(stderr, "[Load] %s backend: %s (CPU threads: %d)\n", label, ggml_backend_name(bp.backend), n_threads);
@@ -146,6 +154,15 @@ static BackendPair backend_init(const char * label) {
 
 // Release a backend reference. Frees GPU + CPU backends when refcount hits 0.
 static void backend_release(ggml_backend_t backend, ggml_backend_t cpu_backend) {
+    // A caller with no backend (m->backend == null) never took a ref -- e.g. a
+    // load that threw before backend_init under ACESTEP_FATAL_THROWS, whose
+    // del_* still runs here. Every caller passes m->backend, so null means "no
+    // ref"; decrementing would corrupt the shared count and could free a backend
+    // another live module still holds. vae/vae-enc open the GGUF before
+    // backend_init, so they are the ones that reach here with a null backend.
+    if (!backend) {
+        return;
+    }
     if (g_backend_refs <= 0) {
         return;
     }
@@ -180,8 +197,7 @@ static ggml_backend_sched_t backend_sched_new(BackendPair bp, int max_nodes) {
 
     ggml_backend_sched_t sched = ggml_backend_sched_new(backends, bufts, n, max_nodes, false, true);
     if (!sched) {
-        fprintf(stderr, "[Load] FATAL: failed to create scheduler\n");
-        exit(1);
+        ace_fatal(1, "[Load] FATAL: failed to create scheduler\n");
     }
     return sched;
 }

src/cond-enc.h

@@ -12,6 +12,7 @@
 //   Pack: cat(lyric, timbre[0:1], text_proj) -> [2048, S_total]
 
 #pragma once
+#include "ace-fatal.h"
 #include "qwen3-enc.h"
 
 // Lyric/Timbre encoder configs
@@ -100,6 +101,7 @@ static bool cond_ggml_load(CondGGML * m, const char * gguf_path) {
         fprintf(stderr, "[Load] FATAL: cannot load %s\n", gguf_path);
         return false;
     }
+    GgufCloser gfc(&gf);
 
     // XL models have encoder_hidden_size in GGUF metadata (2B models omit it).
     // When present, the timbre encoder prepends a learned CLS token.
@@ -185,6 +187,7 @@ static void cond_ggml_forward(CondGGML *           m,
     size_t                  ctx_size = 4096 * ggml_tensor_overhead() + ggml_graph_overhead();
     struct ggml_init_params gp       = { ctx_size, NULL, true };
     struct ggml_context *   ctx      = ggml_init(gp);
+    struct GgmlCtxGuard { ggml_context* p; ~GgmlCtxGuard(){ if(p) ggml_free(p); } } _ctx_guard{ctx};
     struct ggml_cgraph *    gf       = ggml_new_graph_custom(ctx, 8192, false);
 
     // Positions for lyric (bidirectional, 0..S_lyric-1)
@@ -288,8 +291,7 @@ static void cond_ggml_forward(CondGGML *           m,
 
     // Allocate and set inputs
     if (!ggml_backend_sched_alloc_graph(m->sched, gf)) {
-        fprintf(stderr, "[CondEncoder] FATAL: failed to allocate graph\n");
-        exit(1);
+        ace_fatal(1, "[CondEncoder] FATAL: failed to allocate graph\n");
     }
 
     ggml_backend_tensor_set(t_lyric_in, lyric_embed, 0, 1024 * S_lyric * sizeof(float));
@@ -376,7 +378,7 @@ static void cond_ggml_forward(CondGGML *           m,
             S_total);
 
     ggml_backend_sched_reset(m->sched);
-    ggml_free(ctx);
+    // _ctx_guard frees ctx
 }
 
 // Free

src/dit.h

@@ -8,6 +8,7 @@
 // ggml ops used: rms_norm, mul_mat, rope_ext, flash_attn_ext, swiglu_split,
 //                conv_transpose_1d, add, mul, scale, view, reshape, permute.
 
+#include "ace-fatal.h"
 #include "adapter-merge.h"
 #include "backend.h"
 #include "ggml-backend.h"
@@ -147,13 +148,11 @@ static struct ggml_tensor * dit_load_proj_in_w(WeightCtx *         wctx,
                                                int                 P) {
     int64_t idx = gguf_find_tensor(gf.gguf, name.c_str());
     if (idx < 0) {
-        fprintf(stderr, "[GGUF] FATAL: tensor '%s' not found\n", name.c_str());
-        exit(1);
+        ace_fatal(1, "[GGUF] FATAL: tensor '%s' not found\n", name.c_str());
     }
     struct ggml_tensor * src = ggml_get_tensor(gf.meta, name.c_str());
     if (!src) {
-        fprintf(stderr, "[GGUF] FATAL: meta tensor '%s' not found\n", name.c_str());
-        exit(1);
+        ace_fatal(1, "[GGUF] FATAL: meta tensor '%s' not found\n", name.c_str());
     }
     size_t       offset = gguf_get_tensor_offset(gf.gguf, idx);
     const void * raw    = gf.mapping + gf.data_offset + offset;
@@ -186,8 +185,8 @@ static struct ggml_tensor * dit_load_proj_in_w(WeightCtx *         wctx,
         const float * s = (const float *) raw;
         cvt([&](int i) { return s[i]; });
     } else {
-        fprintf(stderr, "[GGUF] FATAL: unsupported type %d for '%s' in proj_in pre-permute\n", src->type, name.c_str());
-        exit(1);
+        ace_fatal(1, "[GGUF] FATAL: unsupported type %d for '%s' in proj_in pre-permute\n", src->type,
+                  name.c_str());
     }
     wctx->pending.push_back({ dst, data, n * sizeof(float), 0 });
     wctx->staging.push_back(std::move(buf));
@@ -204,13 +203,11 @@ static struct ggml_tensor * dit_load_proj_out_w(WeightCtx *         wctx,
                                                 int                 P) {
     int64_t idx = gguf_find_tensor(gf.gguf, name.c_str());
     if (idx < 0) {
-        fprintf(stderr, "[GGUF] FATAL: tensor '%s' not found\n", name.c_str());
-        exit(1);
+        ace_fatal(1, "[GGUF] FATAL: tensor '%s' not found\n", name.c_str());
     }
     struct ggml_tensor * src = ggml_get_tensor(gf.meta, name.c_str());
     if (!src) {
-        fprintf(stderr, "[GGUF] FATAL: meta tensor '%s' not found\n", name.c_str());
-        exit(1);
+        ace_fatal(1, "[GGUF] FATAL: meta tensor '%s' not found\n", name.c_str());
     }
     size_t       offset = gguf_get_tensor_offset(gf.gguf, idx);
     const void * raw    = gf.mapping + gf.data_offset + offset;
@@ -243,9 +240,8 @@ static struct ggml_tensor * dit_load_proj_out_w(WeightCtx *         wctx,
         const float * s = (const float *) raw;
         cvt([&](int i) { return s[i]; });
     } else {
-        fprintf(stderr, "[GGUF] FATAL: unsupported type %d for '%s' in proj_out pre-permute\n", src->type,
-                name.c_str());
-        exit(1);
+        ace_fatal(1, "[GGUF] FATAL: unsupported type %d for '%s' in proj_out pre-permute\n", src->type,
+                  name.c_str());
     }
     wctx->pending.push_back({ dst, data, n * sizeof(float), 0 });
     wctx->staging.push_back(std::move(buf));
@@ -270,6 +266,7 @@ static bool dit_ggml_load(DiTGGML *    m,
         fprintf(stderr, "[Load] FATAL: cannot load %s\n", gguf_path);
         return false;
     }
+    GgufCloser gfc(&gf);
 
     // config from GGUF metadata (all keys required)
     DiTGGMLConfig & cfg         = m->cfg;

src/fsq-detok.h

@@ -79,6 +79,7 @@ static bool detok_ggml_load(DetokGGML * m, const char * gguf_path) {
         fprintf(stderr, "[Load] FATAL: cannot load %s\n", gguf_path);
         return false;
     }
+    GgufCloser gfc(&gf);
 
     // FSQ(2) + embed(2) + special(1) + 2 layers x 11(22) + norm(1) + proj(2) = 30
     wctx_init(&m->wctx, 30);

src/fsq-tok.h

@@ -85,6 +85,7 @@ static bool tok_ggml_load(TokGGML * m, const char * gguf_path) {
         fprintf(stderr, "[Tok] FATAL: cannot load %s\n", gguf_path);
         return false;
     }
+    GgufCloser gfc(&gf);
 
     // proj(2) + embed(2) + special(1) + 2 layers x 11(22) + norm(1) + fsq_in(2) = 30
     wctx_init(&m->wctx, 30);