openclaw - 💡(How to fix) Fix Feature Request: Ollama Media-Understanding Provider [1 comments, 2 participants]

{
  "defaults": {
    "imageModel": {
      "primary": "ollama/qwen3-vl:235b-cloud",
      "fallbacks": []
    }
  }
}

---

[tools] image failed: All image models failed (2): 
ollama/qwen3-vl: No media-understanding provider registered for ollama

---

const PROVIDERS = [groqMediaUnderstandingProvider, deepgramMediaUnderstandingProvider];

function buildMediaUnderstandingRegistry(overrides, cfg) {
  const registry = new Map();
  for (const provider of PROVIDERS) mergeProviderIntoRegistry(registry, provider);
  // Load plugins...
  for (const entry of pluginRegistry?.mediaUnderstandingProviders ?? []) 
    mergeProviderIntoRegistry(registry, entry.provider);
}

function getMediaUnderstandingProvider(id, registry) {
  return registry.get(normalizeMediaProviderId(id));
  // Returns undefined for 'ollama' → throws error
}

---

// extensions/ollama/media-understanding-provider.ts

const DEFAULT_OLLAMA_BASE_URL = "http://127.0.0.1:11434";
const DEFAULT_OLLAMA_VISION_MODEL = "llava";

async function describeOllamaImage(params: DescribeImageParams) {
  const baseUrl = normalizeBaseUrl(params.baseUrl, DEFAULT_OLLAMA_BASE_URL);
  const model = params.model?.trim() || DEFAULT_OLLAMA_VISION_MODEL;
  
  const response = await fetch(`${baseUrl}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model,
      messages: [{
        role: "user",
        content: params.prompt || "Describe this image",
        images: [params.buffer.toString("base64")]
      }]
    })
  });
  
  // Parse response...
}

const ollamaMediaUnderstandingProvider = {
  id: "ollama",
  capabilities: ["image"],
  describeImage: describeOllamaImage,
  describeImages: describeOllamaImages
};