Inspect Structured Import Tasks

Use the AI Studio SDK to list structured import tasks in a workspace page by page, and read status, latest and cumulative row statistics, and error details for a selected task ID.

Task flow

First, paginate through workspace import tasks and filter structured import tasks (configuration type 3) to obtain a target task ID; then query details and row statistics by that ID.

Workspace Connector Service
             │
             ▼
List import tasks page by page and filter structured tasks (config type = 3)
             │
             ▼
       Target Task ID
             │
             ▼
Read task status, latest and cumulative row statistics, and error details

Prerequisites

  1. Complete the endpoint, personal access token, and workspace setup in Use the SDK.

  2. Task ID (when reading task details): run the paginated listing step below, pick a structured import task ID from the output, and export it as TASK_ID.

List structured import tasks

Read workspace import tasks in pages of 100 and print the task ID, name, connector name, and status for structured tasks (configuration type 3). Stop paginating when an empty page is returned or the number of fetched records reaches the total count.

import os

import moi_product_sdk as sdk

client = sdk.new_with_personal_access_token(
    os.environ["PRODUCT_API_BASE_URL"],
    os.environ["PRODUCT_API_KEY"],
)
workspace = client.workspace(os.environ["WORKSPACE_ID"])
service = workspace.connectors()

page = 1
while True:
    result = service.tasks(sdk.with_page(page), sdk.with_page_size(100))
    for item in result.tasks:
        if item.config_type != 3:
            continue
        print(f"id={item.id} name={item.name} connector={item.connector_name} status={item.status}")
    if not result.tasks or page * 100 >= result.total:
        break
    page += 1
package main

import (
	"context"
	"fmt"
	"os"

	sdk "github.com/matrixorigin/matrixflow/sdk/go-sdk"
)

func main() {
	ctx := context.Background()
	client, err := sdk.NewWithPersonalAccessToken(
		os.Getenv("PRODUCT_API_BASE_URL"),
		os.Getenv("PRODUCT_API_KEY"),
	)
	if err != nil {
		panic(err)
	}
	workspace, err := client.Workspace(os.Getenv("WORKSPACE_ID"))
	if err != nil {
		panic(err)
	}
	service := workspace.Connectors()

	for page := 1; ; page++ {
		result, err := service.Tasks(ctx, sdk.WithPage(page), sdk.WithPageSize(100))
		if err != nil {
			panic(err)
		}
		for _, item := range result.GetTasks() {
			if item.GetConfigType() != 3 {
				continue
			}
			fmt.Printf("id=%s name=%s connector=%s status=%d\n", item.GetId(), item.GetName(), item.GetConnectorName(), item.GetStatus())
		}
		if len(result.GetTasks()) == 0 || int64(page*100) >= result.GetTotal() {
			break
		}
	}
}

Read task details and row statistics

Set TASK_ID to a task ID from the previous step and reuse the connector service to read task details, checking total and imported rows, error summaries, and both latest and cumulative row statistics.

task_id = os.environ["TASK_ID"]
info = service.get_task(task_id)
if not info.HasField("task") or info.task.id != task_id:
    raise RuntimeError("Missing task details or mismatched task ID")

task = info.task
print(f"id={task.id} name={task.name} status={task.status}")
print(f"total_rows={task.total_rows} imported_rows={task.imported_rows}")
print(f"error_code={task.error_code} error_summary={task.error_summary}")

for group in ("latest_rows", "cumulative_rows"):
    if not task.HasField(group):
        print(f"{group}=Not provided")
        continue
    stats = getattr(task, group)
    print(f"{group}.complete={stats.complete}")
    for field in ("read_rows", "succeeded_rows", "failed_rows", "skipped_rows"):
        value = getattr(stats, field) if stats.HasField(field) else "Not provided"
        print(f"{group}.{field}={value}")
taskID := os.Getenv("TASK_ID")
info, err := service.GetTask(ctx, taskID)
if err != nil {
	panic(err)
}
task := info.GetTask()
if task == nil || task.GetId() != taskID {
	panic("Missing task details or mismatched task ID")
}

fmt.Printf("id=%s name=%s status=%d\n", task.GetId(), task.GetName(), task.GetStatus())
fmt.Printf("total_rows=%d imported_rows=%d\n", task.GetTotalRows(), task.GetImportedRows())
fmt.Printf("error_code=%s error_summary=%s\n", task.GetErrorCode(), task.GetErrorSummary())

for _, group := range []string{"latest_rows", "cumulative_rows"} {
	stats := task.GetLatestRows()
	if group == "cumulative_rows" {
		stats = task.GetCumulativeRows()
	}
	if stats == nil {
		fmt.Printf("%s=Not provided\n", group)
		continue
	}
	fmt.Printf("%s.complete=%t\n", group, stats.GetComplete())
	fields := []struct {
		name  string
		value *int64
	}{
		{"read_rows", stats.ReadRows},
		{"succeeded_rows", stats.SucceededRows},
		{"failed_rows", stats.FailedRows},
		{"skipped_rows", stats.SkippedRows},
	}
	for _, field := range fields {
		if field.value == nil {
			fmt.Printf("%s.%s=Not provided\n", group, field.name)
		} else {
			fmt.Printf("%s.%s=%d\n", group, field.name, *field.value)
		}
	}
}

Status and statistics fields

Output item

How to interpret

Task ID, name, and connector name

The task ID in the detail response matches TASK_ID. Because tasks can progress while paginating, rely on the detail query for current state.

Task status and error details

Inspect the status code alongside any error code and error summary to determine whether synchronization is progressing or halted.

Total rows and imported rows

Indicate overall task scale and the number of rows written to the destination table.

Latest and cumulative row statistics

Report row counts for the most recent run and across all runs, including a completeness flag and counts for read, succeeded, failed, and skipped rows. An unset field (Not provided) differs from an explicit 0.

Next steps

Last updated on