Commit 30795df

@@ -1,6 +1,7 @@
 #include "llama.h"
 #include "chat.h"
 #include "common.h"
+#include "ggml-backend.h"
 #include <nlohmann/json.hpp>
 #include <algorithm>
 #include <cctype>
@@ -68,6 +69,13 @@ extern "C" {
     fputs(text, stderr);
   }
 
+  static void el_ensure_backend() {
+    if (g_backend) return;
+    llama_log_set(el_log_callback, nullptr);
+    llama_backend_init();
+    g_backend = true;
+  }
+
   static const int EL_MIN_CTX = 4096;
   static const int EL_FALLBACK_CTX = 65536;
 
@@ -104,7 +112,7 @@ extern "C" {
   static llama_context *el_init_context(llama_model *model, int n_ctx, int n_threads) {
     int32_t trained_max = llama_model_n_ctx_train(model);
     uint32_t requested = n_ctx > 0 ? (uint32_t) n_ctx
-      : trained_max > 0 ? (uint32_t) trained_max
+      : trained_max > 0 ? std::min((uint32_t) trained_max, (uint32_t) EL_FALLBACK_CTX)
       : (uint32_t) EL_FALLBACK_CTX;
     llama_context_params cp = llama_context_default_params();
     cp.n_ctx = requested;
@@ -123,12 +131,17 @@ extern "C" {
     return nullptr;
   }
 
+  long el_gpu_free_memory() {
+    el_ensure_backend();
+    ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
+    if (!dev) return -1;
+    size_t free_mem = 0, total_mem = 0;
+    ggml_backend_dev_memory(dev, &free_mem, &total_mem);
+    return (long) free_mem;
+  }
+
   void *el_open(const char *path, int n_gpu_layers, int n_ctx, int n_threads, float temp, int seed) {
-    if (!g_backend) {
-      llama_log_set(el_log_callback, nullptr);
-      llama_backend_init();
-      g_backend = true;
-    }
+    el_ensure_backend();
 
     llama_model_params mp = llama_model_default_params();
     mp.n_gpu_layers = n_gpu_layers;

@@ -127,7 +127,7 @@ def link_shim(libdir)
       "-I", File.join(VENDOR, "vendor"),   # nlohmann/json.hpp
       File.join(EXT_DIR, "elelem.cpp"),
       "-o", shim,
-      "-L", libdir, "-lllama-common", "-lllama",
+      "-L", libdir, "-lllama-common", "-lllama", "-lggml", "-lggml-base",
       *link_flags)
 end
 

@@ -15,13 +15,19 @@ module Elelem
         "cpu"
       end
 
+      def self.gpu_free_memory
+        bytes = functions[:gpu_free_memory].call
+        bytes.negative? ? nil : bytes
+      end
+
       def self.functions
         @functions ||= begin
           lib = Fiddle.dlopen(SHIM)
           {
             open: Fiddle::Function.new(lib["el_open"], [V, I, I, I, F, I], V),
             generate: Fiddle::Function.new(lib["el_generate"], [V, V, V, I, V, V], V, need_gvl: true),
-            close: Fiddle::Function.new(lib["el_close"], [V], Fiddle::TYPE_VOID)
+            close: Fiddle::Function.new(lib["el_close"], [V], Fiddle::TYPE_VOID),
+            gpu_free_memory: Fiddle::Function.new(lib["el_gpu_free_memory"], [], Fiddle::TYPE_LONG)
           }
         end
       end

@@ -9,8 +9,6 @@ require_relative "llama/provider"
 
 Elelem.configure do |config|
   config.provider(:llama) do |model: nil, **|
-    gpu = %w[vulkan cuda metal].include?(Elelem::Llama::Provider.backend)
-
     default_model = lambda {
       dir = File.expand_path("~/.agents/models")
       link = File.join(dir, "default.gguf")
@@ -25,11 +23,22 @@ Elelem.configure do |config|
       end
     }
 
+    model_path = model || ENV.fetch("ELELEM_GGUF_MODEL") { default_model.call }
+
+    default_gpu_layers = lambda {
+      backend = Elelem::Llama::Provider.backend
+      next "0" unless %w[vulkan cuda metal].include?(backend)
+      next "0" if backend == "metal" && !RUBY_PLATFORM.include?("arm64")
+
+      free = Elelem::Llama::Provider.gpu_free_memory
+      free && File.size(model_path) < free * 0.8 ? "999" : "0"
+    }
+
     Elelem::Llama::Provider.new(
-      model: model || ENV.fetch("ELELEM_GGUF_MODEL") { default_model.call },
-      n_ctx: 0,
+      model: model_path,
+      n_ctx: Integer(ENV.fetch("ELELEM_GGUF_N_CTX", "0")),
       n_threads: 0,
-      n_gpu_layers: gpu ? 999 : 0,
+      n_gpu_layers: Integer(ENV.fetch("ELELEM_GGUF_N_GPU_LAYERS") { default_gpu_layers.call }),
       max_tokens: Integer(ENV.fetch("ELELEM_GGUF_MAX_TOKENS", "0")),
       temp: Float(ENV.fetch("ELELEM_GGUF_TEMP", "0.7")),
       seed: Integer(ENV.fetch("ELELEM_GGUF_SEED", "-1"))