Publish and Subscribe to Data¶
Publish tables from a source database in a source workspace, and query and subscribe to that publication in a destination workspace with the SDK. Publishing and subscribing run in separate workspaces and return independent resource handles and states.
Task flow¶
Source workspace checks shareable tables & publishes ──▶ Target workspace lists subscriptions ──▶ Subscribe & verify status
Prerequisites¶
Configure the client and personal access token (
PRODUCT_API_BASE_URL,PRODUCT_API_KEY) by following Use the SDK, and obtain both the source workspace ID (WORKSPACE_ID) and the destination workspace ID (TARGET_WORKSPACE_ID).Source database ID and table ID: Obtain the source database ID (
SOURCE_DATABASE_ID) and the table ID to publish (SOURCE_TABLE_ID) via Manage Catalogs, Databases, and Tables; if you already have the source database ID, you can also look up shareable table IDs in Step 1.Publication name, subscription name, and remark: Choose a publication name, a subscription name in the destination workspace, and a business remark. This example uses
sdk_sales_publishandsdk_sales_subscription; adjust both names for your workload before running.
Export the cross-workspace sharing parameters in your terminal:
export TARGET_WORKSPACE_ID='<destination workspace ID>'
export SOURCE_DATABASE_ID='<source database ID>'
export SOURCE_TABLE_ID='<table ID to publish>'
1. Create a data publication in the source workspace¶
List the shareable tables in the source database to confirm that your target table is eligible, then specify the receiving workspace IDs and create the publication. Once created, you can use the returned publication handle to update the table scope, destination workspaces, or remark, or to delete the publication.
import os
import moi_product_sdk as sdk
client = sdk.new_with_personal_access_token(
os.environ["PRODUCT_API_BASE_URL"], os.environ["PRODUCT_API_KEY"]
)
source_workspace = client.workspace(os.environ["WORKSPACE_ID"])
target_workspace_id = os.environ["TARGET_WORKSPACE_ID"]
source_database_id = os.environ["SOURCE_DATABASE_ID"]
source_table_id = os.environ["SOURCE_TABLE_ID"]
source_tables = source_workspace.data_share_source_tables(source_database_id)
if source_table_id not in {item.id for item in source_tables.list}:
raise ValueError("source table is not in the shareable table list")
table_scope = sdk.DataShareScope(
mode="selected",
object_ids=[source_table_id],
)
publish, created = source_workspace.create_data_share_publish(
"sdk_sales_publish",
source_database_id,
table_scope,
[target_workspace_id],
sdk.with_data_share_remark("SDK data sharing example"),
)
print("Publication ID:", publish.id, "Publication name:", created.name)
package main
import (
"context"
"fmt"
"os"
sdk "github.com/matrixorigin/matrixflow/sdk/go-sdk"
)
func main() {
ctx := context.Background()
client, err := sdk.NewWithPersonalAccessToken(
os.Getenv("PRODUCT_API_BASE_URL"), os.Getenv("PRODUCT_API_KEY"),
)
if err != nil {
panic(err)
}
sourceWorkspace, err := client.Workspace(os.Getenv("WORKSPACE_ID"))
if err != nil {
panic(err)
}
targetWorkspaceID := os.Getenv("TARGET_WORKSPACE_ID")
sourceDatabaseID := os.Getenv("SOURCE_DATABASE_ID")
sourceTableID := os.Getenv("SOURCE_TABLE_ID")
sourceTables, err := sourceWorkspace.DataShareSourceTables(ctx, sourceDatabaseID)
if err != nil {
panic(err)
}
shareable := false
for _, table := range sourceTables.GetList() {
if table.GetId() == sourceTableID {
shareable = true
break
}
}
if !shareable {
panic(fmt.Sprintf("source table %q is not in the shareable table list", sourceTableID))
}
tableScope := &sdk.DataShareScope{
Mode: "selected",
ObjectIds: []string{sourceTableID},
}
publish, created, err := sourceWorkspace.CreateDataSharePublish(
ctx,
"sdk_sales_publish",
sourceDatabaseID,
tableScope,
[]string{targetWorkspaceID},
sdk.WithDataShareRemark("SDK data sharing example"),
)
if err != nil {
panic(err)
}
fmt.Println("Publication ID:", publish.ID(), "Publication name:", created.GetName())
}
2. Locate and accept the subscription in the target workspace¶
Construct a handle for the destination workspace, list visible subscriptions to find the entry matching the publication name, and subscribe to it, verifying that the returned status is subscribed.
target_workspace = client.workspace(target_workspace_id)
subscriptions = target_workspace.data_share().subscriptions()
subscription_id = next(
(item.id for item in subscriptions.list if item.pub_name == created.name),
"",
)
if not subscription_id:
raise RuntimeError("subscription entry was not found in the target workspace")
subscription = target_workspace.data_share_subscription(subscription_id)
subscribed = subscription.subscribe("sdk_sales_subscription")
if subscribed.status != "subscribed":
raise RuntimeError(f"unexpected subscription status: {subscribed.status}")
print("Subscription completed, status:", subscribed.status)
targetWorkspace, err := client.Workspace(targetWorkspaceID)
if err != nil {
panic(err)
}
subscriptions, err := targetWorkspace.DataShare().Subscriptions(ctx)
if err != nil {
panic(err)
}
subscriptionID := ""
for _, item := range subscriptions.GetList() {
if item.GetPubName() == created.GetName() {
subscriptionID = item.GetId()
break
}
}
if subscriptionID == "" {
panic(fmt.Sprintf("subscription entry for %q was not found in the target workspace", created.GetName()))
}
subscription, err := targetWorkspace.DataShareSubscription(subscriptionID)
if err != nil {
panic(err)
}
subscribed, err := subscription.Subscribe(ctx, "sdk_sales_subscription")
if err != nil {
panic(err)
}
if subscribed.GetStatus() != "subscribed" {
panic(fmt.Sprintf("unexpected subscription status: %s", subscribed.GetStatus()))
}
fmt.Println("Subscription completed, status:", subscribed.GetStatus())
Next steps¶
Manage Catalogs, Databases, and Tables: Inspect databases and tables created from subscriptions in the destination workspace.
Run SQL Queries: Query subscribed tables in the destination workspace.