Send Image Messages

Send an image stored in a workspace volume together with a text prompt to an agent, then inspect the returned task state and recognition output. Image understanding requires a model that supports visual input.

Task flow

Workspace volume image (volume_id + file_id) + vision model
                            │
                            ▼
    Build A2A message with file part and catalog reference
                            │
                            ▼
            Submit request and obtain task_id
                            │
                            ▼
      Verify artifacts once task state reaches completed

Prerequisites

  1. Complete the endpoint, personal access token, and workspace setup in Use the SDK.

  2. Select an agent in the current workspace and record its ID by following Discover and Select Agents.

  3. Choose a model that supports visual input: query declared model names via workspace.agents().resource_get("llm", "/models", sdk.with_query_param("type", "vision")) in Python or workspace.Agents().ResourceGet(ctx, "llm", "/models", sdk.WithQueryParam("type", "vision")) in Go, and confirm that the backend service supports vision input (see Inspect Model Backends and Endpoints).

  4. Upload a PNG image to a volume in the current workspace (or choose an existing readable image) following Manage Volumes and Files, and record its positive integer root volume ID and file ID.

Set the runtime variables in your terminal:

export AGENT_ID='<agent-id>'
export AGENT_MODEL='<vision-capable-model-name>'
export IMAGE_VOLUME_ID='<root-volume-id>'
export IMAGE_FILE_ID='<image-file-id>'

Send an image and inspect results

The example combines a text part and a moi://files/<file_id> file part in one A2A message and declares the matching workspace volume file under metadata.files. When sending another image format, update mimeType and the display filename together so the file content, URI, and catalog reference point to the same file.

import json
import os
import uuid

import moi_product_sdk as sdk

client = sdk.new_with_personal_access_token(
    os.environ["PRODUCT_API_BASE_URL"],
    os.environ["PRODUCT_API_KEY"],
)
workspace = client.workspace(os.environ["WORKSPACE_ID"])
agent = workspace.agent_by_id(os.environ["AGENT_ID"])

file_id = os.environ["IMAGE_FILE_ID"]
volume_id = int(os.environ["IMAGE_VOLUME_ID"])
message_id = str(uuid.uuid4())
print("message_id:", message_id)

params = {
    "model": os.environ["AGENT_MODEL"],
    "message": {
        "kind": "message",
        "role": "user",
        "messageId": message_id,
        "parts": [
            {
                "kind": "text",
                "text": "Describe the visible labels and values in this image.",
            },
            {
                "kind": "file",
                "file": {
                    "uri": "moi://files/" + file_id,
                    "mimeType": "image/png",
                    "name": "chart.png",
                },
                "metadata": {"file_id": file_id},
            },
        ],
    },
    "metadata": {
        "files": [
            {
                "catalog_file": {
                    "workspace_id": os.environ["WORKSPACE_ID"],
                    "volume_id": volume_id,
                    "file_id": file_id,
                }
            }
        ]
    },
}

try:
    result = agent.a2a(
        sdk.A2ARequest("message/send", id_=str(uuid.uuid4()), params=params)
    )
except sdk.Error as error:
    if error.details.get("task_id"):
        print("task_id:", error.details["task_id"])
    raise

payload = sdk.agent_a2a_payload(result)
if payload.get("kind") != "task":
    raise RuntimeError("Expected a task response; inspect returned message payload")

print("task_id:", payload.get("id"))
print("contextId:", payload.get("contextId"))
print("state:", payload.get("status", {}).get("state"))
print(
    "status message:",
    json.dumps(payload.get("status", {}).get("message"), ensure_ascii=False),
)
for artifact in payload.get("artifacts", []):
    print("artifactId:", artifact.get("artifactId"))
    print("parts:", json.dumps(artifact.get("parts", []), ensure_ascii=False))
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"
	"time"

	sdk "github.com/matrixorigin/matrixflow/sdk/go-sdk"
)

func main() {
	ctx := context.Background()
	client, err := sdk.NewWithPersonalAccessToken(
		os.Getenv("PRODUCT_API_BASE_URL"),
		os.Getenv("PRODUCT_API_KEY"),
	)
	if err != nil {
		panic(err)
	}
	defer client.Close()

	workspace, err := client.Workspace(os.Getenv("WORKSPACE_ID"))
	if err != nil {
		panic(err)
	}
	agent, err := workspace.AgentByID(os.Getenv("AGENT_ID"))
	if err != nil {
		panic(err)
	}

	fileID := os.Getenv("IMAGE_FILE_ID")
	volumeID, err := strconv.ParseInt(os.Getenv("IMAGE_VOLUME_ID"), 10, 64)
	if err != nil || volumeID <= 0 {
		panic("IMAGE_VOLUME_ID must be a positive integer")
	}
	messageID := fmt.Sprintf("image-%d", time.Now().UnixNano())
	fmt.Println("message_id:", messageID)

	params, err := json.Marshal(map[string]any{
		"model": os.Getenv("AGENT_MODEL"),
		"message": map[string]any{
			"kind":      "message",
			"role":      "user",
			"messageId": messageID,
			"parts": []any{
				map[string]any{
					"kind": "text",
					"text": "Describe the visible labels and values in this image.",
				},
				map[string]any{
					"kind": "file",
					"file": map[string]any{
						"uri":      "moi://files/" + fileID,
						"mimeType": "image/png",
						"name":     "chart.png",
					},
					"metadata": map[string]any{"file_id": fileID},
				},
			},
		},
		"metadata": map[string]any{
			"files": []any{
				map[string]any{
					"catalog_file": map[string]any{
						"workspace_id": os.Getenv("WORKSPACE_ID"),
						"volume_id":    volumeID,
						"file_id":      fileID,
					},
				},
			},
		},
	})
	if err != nil {
		panic(err)
	}

	result, err := agent.A2A(ctx, sdk.A2ARequest{
		JSONRPC: "2.0",
		ID:      fmt.Sprintf("request-%d", time.Now().UnixNano()),
		Method:  sdk.A2AMethodSend,
		Params:  params,
	})
	if err != nil {
		var apiError *sdk.Error
		if errors.As(err, &apiError) && apiError.Details["task_id"] != "" {
			fmt.Println("task_id:", apiError.Details["task_id"])
		}
		panic(err)
	}

	payload, err := sdk.AgentA2APayload(result)
	if err != nil {
		panic(err)
	}
	if payload["kind"] != "task" {
		panic("Expected a task response; inspect returned message payload")
	}
	fmt.Println("task_id:", payload["id"], "contextId:", payload["contextId"])
	status, _ := payload["status"].(map[string]any)
	fmt.Println("state:", status["state"])
	message, _ := json.Marshal(status["message"])
	fmt.Println("status message:", string(message))
	artifacts, _ := json.Marshal(payload["artifacts"])
	fmt.Println("artifacts:", string(artifacts))
}

Limitations and notes

  • Inspect recognition output in artifacts after the task state reaches completed; handle input-required or auth-required states before expecting final artifacts.

  • If waiting times out and the error details include task_id, query that task ID first to avoid submitting duplicate image requests.

  • catalog_file must reference a readable file in a volume of the current workspace; local filesystem paths, agent artifact IDs, or file IDs from other workspaces cannot be used in its place.

Next steps

Last updated on