Publish and Subscribe to Data

Publish tables from a source database in a source workspace, and query and subscribe to that publication in a destination workspace with the SDK. Publishing and subscribing run in separate workspaces and return independent resource handles and states.

Task flow

Source workspace checks shareable tables & publishes ──▶ Target workspace lists subscriptions ──▶ Subscribe & verify status

Prerequisites

  1. Configure the client and personal access token (PRODUCT_API_BASE_URL, PRODUCT_API_KEY) by following Use the SDK, and obtain both the source workspace ID (WORKSPACE_ID) and the destination workspace ID (TARGET_WORKSPACE_ID).

  2. Source database ID and table ID: Obtain the source database ID (SOURCE_DATABASE_ID) and the table ID to publish (SOURCE_TABLE_ID) via Manage Catalogs, Databases, and Tables; if you already have the source database ID, you can also look up shareable table IDs in Step 1.

  3. Publication name, subscription name, and remark: Choose a publication name, a subscription name in the destination workspace, and a business remark. This example uses sdk_sales_publish and sdk_sales_subscription; adjust both names for your workload before running.

Export the cross-workspace sharing parameters in your terminal:

export TARGET_WORKSPACE_ID='<destination workspace ID>'
export SOURCE_DATABASE_ID='<source database ID>'
export SOURCE_TABLE_ID='<table ID to publish>'

1. Create a data publication in the source workspace

List the shareable tables in the source database to confirm that your target table is eligible, then specify the receiving workspace IDs and create the publication. Once created, you can use the returned publication handle to update the table scope, destination workspaces, or remark, or to delete the publication.

import os
import moi_product_sdk as sdk

client = sdk.new_with_personal_access_token(
    os.environ["PRODUCT_API_BASE_URL"], os.environ["PRODUCT_API_KEY"]
)
source_workspace = client.workspace(os.environ["WORKSPACE_ID"])
target_workspace_id = os.environ["TARGET_WORKSPACE_ID"]
source_database_id = os.environ["SOURCE_DATABASE_ID"]
source_table_id = os.environ["SOURCE_TABLE_ID"]

source_tables = source_workspace.data_share_source_tables(source_database_id)
if source_table_id not in {item.id for item in source_tables.list}:
    raise ValueError("source table is not in the shareable table list")

table_scope = sdk.DataShareScope(
    mode="selected",
    object_ids=[source_table_id],
)
publish, created = source_workspace.create_data_share_publish(
    "sdk_sales_publish",
    source_database_id,
    table_scope,
    [target_workspace_id],
    sdk.with_data_share_remark("SDK data sharing example"),
)
print("Publication ID:", publish.id, "Publication name:", created.name)
package main

import (
	"context"
	"fmt"
	"os"

	sdk "github.com/matrixorigin/matrixflow/sdk/go-sdk"
)

func main() {
	ctx := context.Background()
	client, err := sdk.NewWithPersonalAccessToken(
		os.Getenv("PRODUCT_API_BASE_URL"), os.Getenv("PRODUCT_API_KEY"),
	)
	if err != nil {
		panic(err)
	}
	sourceWorkspace, err := client.Workspace(os.Getenv("WORKSPACE_ID"))
	if err != nil {
		panic(err)
	}
	targetWorkspaceID := os.Getenv("TARGET_WORKSPACE_ID")
	sourceDatabaseID := os.Getenv("SOURCE_DATABASE_ID")
	sourceTableID := os.Getenv("SOURCE_TABLE_ID")

	sourceTables, err := sourceWorkspace.DataShareSourceTables(ctx, sourceDatabaseID)
	if err != nil {
		panic(err)
	}
	shareable := false
	for _, table := range sourceTables.GetList() {
		if table.GetId() == sourceTableID {
			shareable = true
			break
		}
	}
	if !shareable {
		panic(fmt.Sprintf("source table %q is not in the shareable table list", sourceTableID))
	}

	tableScope := &sdk.DataShareScope{
		Mode:      "selected",
		ObjectIds: []string{sourceTableID},
	}
	publish, created, err := sourceWorkspace.CreateDataSharePublish(
		ctx,
		"sdk_sales_publish",
		sourceDatabaseID,
		tableScope,
		[]string{targetWorkspaceID},
		sdk.WithDataShareRemark("SDK data sharing example"),
	)
	if err != nil {
		panic(err)
	}
	fmt.Println("Publication ID:", publish.ID(), "Publication name:", created.GetName())
}

2. Locate and accept the subscription in the target workspace

Construct a handle for the destination workspace, list visible subscriptions to find the entry matching the publication name, and subscribe to it, verifying that the returned status is subscribed.

target_workspace = client.workspace(target_workspace_id)
subscriptions = target_workspace.data_share().subscriptions()
subscription_id = next(
    (item.id for item in subscriptions.list if item.pub_name == created.name),
    "",
)
if not subscription_id:
    raise RuntimeError("subscription entry was not found in the target workspace")

subscription = target_workspace.data_share_subscription(subscription_id)
subscribed = subscription.subscribe("sdk_sales_subscription")
if subscribed.status != "subscribed":
    raise RuntimeError(f"unexpected subscription status: {subscribed.status}")
print("Subscription completed, status:", subscribed.status)
targetWorkspace, err := client.Workspace(targetWorkspaceID)
if err != nil {
	panic(err)
}
subscriptions, err := targetWorkspace.DataShare().Subscriptions(ctx)
if err != nil {
	panic(err)
}
subscriptionID := ""
for _, item := range subscriptions.GetList() {
	if item.GetPubName() == created.GetName() {
		subscriptionID = item.GetId()
		break
	}
}
if subscriptionID == "" {
	panic(fmt.Sprintf("subscription entry for %q was not found in the target workspace", created.GetName()))
}

subscription, err := targetWorkspace.DataShareSubscription(subscriptionID)
if err != nil {
	panic(err)
}
subscribed, err := subscription.Subscribe(ctx, "sdk_sales_subscription")
if err != nil {
	panic(err)
}
if subscribed.GetStatus() != "subscribed" {
	panic(fmt.Sprintf("unexpected subscription status: %s", subscribed.GetStatus()))
}
fmt.Println("Subscription completed, status:", subscribed.GetStatus())

After subscribing, re-query the subscription list in the destination workspace to verify the subscription status and the generated target database details. Before publishing data across workspaces, double-check the selected table scope, receiving workspace IDs, and access permissions.

Next steps

Last updated on