diff --git a/coordinator/capture_write_lease.go b/coordinator/capture_write_lease.go index df7271d426..236cdad6c9 100644 --- a/coordinator/capture_write_lease.go +++ b/coordinator/capture_write_lease.go @@ -27,13 +27,19 @@ import ( ) const ( - witnessNonceSize = 16 - witnessChallengeTimeout = time.Second + witnessNonceSize = 16 + witnessChallengeTimeout = time.Second + nodeResourceUsageStaleThreshold = 5 * time.Second ) type captureLeaseNodeState struct { - nodeEpoch uint64 - lastRequestSeq uint64 + nodeEpoch uint64 + lastRequestSeq uint64 + resourceUsageProtocolVersion uint32 + eventStoreWriteBytes uint64 + eventStoreWriteBytesPerSecond uint64 + resourceUsageRateAvailable bool + resourceUsageUpdated time.Time } type pendingWitnessChallenge struct { @@ -54,6 +60,7 @@ type captureWriteLeaseController struct { nonce func([]byte) (int, error) nodes map[node.ID]*captureLeaseNodeState + activeNodes map[node.ID]struct{} p2pCapableNodes map[node.ID]struct{} p2pLeaseEnabled bool pendingWitness *pendingWitnessChallenge @@ -67,6 +74,7 @@ func newCaptureWriteLeaseController(version int64, selfNodeID node.ID) *captureW now: time.Now, nonce: rand.Read, nodes: make(map[node.ID]*captureLeaseNodeState), + activeNodes: make(map[node.ID]struct{}), p2pCapableNodes: make(map[node.ID]struct{}), } } @@ -82,6 +90,11 @@ func (c *captureWriteLeaseController) observeNodeCapability(id node.ID, version // updateClusterMode enables P2P only when every active capture has reported // support for the current protocol. A missing capability is treated as legacy. func (c *captureWriteLeaseController) updateClusterMode(activeNodes []node.ID) { + c.activeNodes = make(map[node.ID]struct{}, len(activeNodes)) + for _, id := range activeNodes { + c.activeNodes[id] = struct{}{} + } + p2pEnabled := len(activeNodes) > 0 for _, id := range activeNodes { if _, ok := c.p2pCapableNodes[id]; !ok { @@ -120,6 +133,29 @@ func (c *captureWriteLeaseController) handleHeartbeat( return nil } state.lastRequestSeq = heartbeat.GetWriteLeaseRequestSeq() + state.resourceUsageProtocolVersion = heartbeat.GetNodeResourceUsageProtocolVersion() + usage := heartbeat.GetNodeResourceUsage() + if state.resourceUsageProtocolVersion == heartbeatpb.CurrentNodeResourceUsageProtocolVersion && usage != nil { + now := c.now() + writeBytes := usage.GetEventStoreWriteBytes() + if !state.resourceUsageUpdated.IsZero() && now.After(state.resourceUsageUpdated) { + if writeBytes >= state.eventStoreWriteBytes { + state.eventStoreWriteBytesPerSecond = uint64( + float64(writeBytes-state.eventStoreWriteBytes) / + now.Sub(state.resourceUsageUpdated).Seconds()) + state.resourceUsageRateAvailable = true + } else { + state.resourceUsageRateAvailable = false + } + } + if state.resourceUsageUpdated.IsZero() || now.After(state.resourceUsageUpdated) { + state.eventStoreWriteBytes = writeBytes + state.resourceUsageUpdated = now + } + } else { + state.resourceUsageUpdated = time.Time{} + state.resourceUsageRateAvailable = false + } messages := c.handleWitnessAck(from, heartbeat) if from != c.selfNodeID { @@ -248,20 +284,61 @@ func (c *captureWriteLeaseController) newGrant( if c.p2pLeaseEnabled { leaseDurationMs = uint64(writelease.P2PLeaseDuration.Milliseconds()) } + nodeResourceUsages, nodeResourceUsageStatus := c.nodeResourceUsageSnapshot() return messaging.NewSingleTargetMessage( target, messaging.MaintainerManagerTopic, &heartbeatpb.NodeHeartbeatResponse{ - CoordinatorVersion: c.coordinatorVersion, - TargetNodeEpoch: targetNodeEpoch, - RequestSeq: requestSeq, - LeaseDurationMs: leaseDurationMs, + CoordinatorVersion: c.coordinatorVersion, + TargetNodeEpoch: targetNodeEpoch, + RequestSeq: requestSeq, + LeaseDurationMs: leaseDurationMs, + NodeResourceUsages: nodeResourceUsages, + NodeResourceUsageStatus: nodeResourceUsageStatus, }, ) } +func (c *captureWriteLeaseController) nodeResourceUsageSnapshot() ( + []*heartbeatpb.NodeResourceUsage, + heartbeatpb.NodeResourceUsageStatus, +) { + if len(c.activeNodes) == 0 { + return nil, heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED + } + for nodeID := range c.activeNodes { + state := c.nodes[nodeID] + if state == nil || + state.resourceUsageProtocolVersion != heartbeatpb.CurrentNodeResourceUsageProtocolVersion { + return nil, heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED + } + } + + now := c.now() + nodeIDs := make([]node.ID, 0, len(c.activeNodes)) + for nodeID := range c.activeNodes { + state := c.nodes[nodeID] + if !state.resourceUsageRateAvailable || state.resourceUsageUpdated.IsZero() || + now.Sub(state.resourceUsageUpdated) > nodeResourceUsageStaleThreshold { + return nil, heartbeatpb.NodeResourceUsageStatus_INCOMPLETE + } + nodeIDs = append(nodeIDs, nodeID) + } + slices.Sort(nodeIDs) + + result := make([]*heartbeatpb.NodeResourceUsage, 0, len(nodeIDs)) + for _, nodeID := range nodeIDs { + result = append(result, &heartbeatpb.NodeResourceUsage{ + NodeId: nodeID.String(), + EventStoreWriteBytesPerSecond: c.nodes[nodeID].eventStoreWriteBytesPerSecond, + }) + } + return result, heartbeatpb.NodeResourceUsageStatus_AVAILABLE +} + func (c *captureWriteLeaseController) removeNode(id node.ID) { delete(c.nodes, id) + delete(c.activeNodes, id) delete(c.p2pCapableNodes, id) if c.pendingWitness != nil && (c.pendingWitness.witnessNodeID == id || id == c.selfNodeID) { diff --git a/coordinator/capture_write_lease_test.go b/coordinator/capture_write_lease_test.go index 09a94e1881..fdf96db31c 100644 --- a/coordinator/capture_write_lease_test.go +++ b/coordinator/capture_write_lease_test.go @@ -51,6 +51,94 @@ func TestCaptureWriteLeaseGrantsRemoteNode(t *testing.T) { require.Equal(t, uint64(12), requireWriteLeaseResponse(t, messages[0]).TargetNodeEpoch) } +func TestCaptureWriteLeaseSharesFreshNodeResourceUsage(t *testing.T) { + now := time.Unix(100, 0) + controller := newCaptureWriteLeaseController(10, node.ID("coordinator")) + controller.now = func() time.Time { return now } + enableP2PForNodes(controller, node.ID("capture-1"), node.ID("capture-2")) + + capture2Heartbeat := newWriteLeaseHeartbeat(21, 1) + capture2Heartbeat.NodeResourceUsage = &heartbeatpb.NodeResourceUsage{ + EventStoreWriteBytes: 1000, + } + controller.handleHeartbeat(node.ID("capture-2"), capture2Heartbeat, nil) + + capture1Heartbeat := newWriteLeaseHeartbeat(11, 1) + capture1Heartbeat.NodeResourceUsage = &heartbeatpb.NodeResourceUsage{ + EventStoreWriteBytes: 100, + } + messages := controller.handleHeartbeat(node.ID("capture-1"), capture1Heartbeat, nil) + require.Len(t, messages, 1) + require.Equal(t, + heartbeatpb.NodeResourceUsageStatus_INCOMPLETE, + requireWriteLeaseResponse(t, messages[0]).NodeResourceUsageStatus) + + // Each node's rate is calculated from its own heartbeat interval. + now = now.Add(time.Second) + capture2Heartbeat.WriteLeaseRequestSeq = 2 + capture2Heartbeat.NodeResourceUsage.EventStoreWriteBytes = 3000 + controller.handleHeartbeat(node.ID("capture-2"), capture2Heartbeat, nil) + capture1Heartbeat.WriteLeaseRequestSeq = 2 + capture1Heartbeat.NodeResourceUsage.EventStoreWriteBytes = 200 + messages = controller.handleHeartbeat(node.ID("capture-1"), capture1Heartbeat, nil) + require.Len(t, messages, 1) + require.Equal(t, + heartbeatpb.NodeResourceUsageStatus_AVAILABLE, + requireWriteLeaseResponse(t, messages[0]).NodeResourceUsageStatus) + require.Equal(t, []*heartbeatpb.NodeResourceUsage{ + {NodeId: "capture-1", EventStoreWriteBytesPerSecond: 100}, + {NodeId: "capture-2", EventStoreWriteBytesPerSecond: 2000}, + }, requireWriteLeaseResponse(t, messages[0]).NodeResourceUsages) + + // Only capture-1 reports again. A response reuses capture-2's last valid + // rate instead of interpreting the unchanged snapshot as zero traffic. + now = now.Add(time.Second) + capture1Heartbeat.WriteLeaseRequestSeq = 3 + capture1Heartbeat.NodeResourceUsage.EventStoreWriteBytes = 300 + messages = controller.handleHeartbeat(node.ID("capture-1"), capture1Heartbeat, nil) + require.Equal(t, []*heartbeatpb.NodeResourceUsage{ + {NodeId: "capture-1", EventStoreWriteBytesPerSecond: 100}, + {NodeId: "capture-2", EventStoreWriteBytesPerSecond: 2000}, + }, requireWriteLeaseResponse(t, messages[0]).NodeResourceUsages) + + // A fresh report must not keep another node's stale sample in the cluster + // snapshot. All nodes support reporting, so this is an interruption rather + // than a rolling-upgrade fallback. + now = now.Add(nodeResourceUsageStaleThreshold + time.Nanosecond) + capture1Heartbeat.WriteLeaseRequestSeq = 4 + capture1Heartbeat.NodeResourceUsage.EventStoreWriteBytes = 400 + messages = controller.handleHeartbeat(node.ID("capture-1"), capture1Heartbeat, nil) + require.Len(t, messages, 1) + response := requireWriteLeaseResponse(t, messages[0]) + require.Equal(t, + heartbeatpb.NodeResourceUsageStatus_INCOMPLETE, + response.NodeResourceUsageStatus) + require.Empty(t, response.NodeResourceUsages) + + // Missing usage means the sender no longer supports or cannot provide the + // counter, so its previous value is removed immediately. + capture1Heartbeat.WriteLeaseRequestSeq = 5 + capture1Heartbeat.NodeResourceUsage = nil + messages = controller.handleHeartbeat(node.ID("capture-1"), capture1Heartbeat, nil) + require.Len(t, messages, 1) + response = requireWriteLeaseResponse(t, messages[0]) + require.Equal(t, + heartbeatpb.NodeResourceUsageStatus_INCOMPLETE, + response.NodeResourceUsageStatus) + require.Empty(t, response.NodeResourceUsages) + + // A node that does not declare the resource protocol is a rolling-upgrade + // compatibility case, distinct from interrupted telemetry. + capture1Heartbeat.WriteLeaseRequestSeq = 6 + capture1Heartbeat.NodeResourceUsageProtocolVersion = heartbeatpb.LegacyNodeResourceUsageProtocolVersion + messages = controller.handleHeartbeat(node.ID("capture-1"), capture1Heartbeat, nil) + require.Len(t, messages, 1) + response = requireWriteLeaseResponse(t, messages[0]) + require.Equal(t, + heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED, + response.NodeResourceUsageStatus) +} + func TestCaptureWriteLeaseRequiresRemoteWitnessForCoordinatorNode(t *testing.T) { now := time.Unix(100, 0) controller := newCaptureWriteLeaseController(10, node.ID("coordinator")) @@ -246,10 +334,11 @@ func TestCaptureWriteLeaseRejectsInvalidHeartbeatAndLateWitness(t *testing.T) { func newWriteLeaseHeartbeat(nodeEpoch, requestSeq uint64) *heartbeatpb.NodeHeartbeat { return &heartbeatpb.NodeHeartbeat{ - Liveness: heartbeatpb.NodeLiveness_ALIVE, - NodeEpoch: nodeEpoch, - WriteLeaseRequestSeq: requestSeq, - WriteLeaseProtocolVersion: heartbeatpb.CurrentWriteLeaseProtocolVersion, + Liveness: heartbeatpb.NodeLiveness_ALIVE, + NodeEpoch: nodeEpoch, + WriteLeaseRequestSeq: requestSeq, + WriteLeaseProtocolVersion: heartbeatpb.CurrentWriteLeaseProtocolVersion, + NodeResourceUsageProtocolVersion: heartbeatpb.CurrentNodeResourceUsageProtocolVersion, } } diff --git a/downstreamadapter/dispatchermanager/dispatcher_manager.go b/downstreamadapter/dispatchermanager/dispatcher_manager.go index 42fffd1f79..f773874ee3 100644 --- a/downstreamadapter/dispatchermanager/dispatcher_manager.go +++ b/downstreamadapter/dispatchermanager/dispatcher_manager.go @@ -924,7 +924,6 @@ func (e *DispatcherManager) aggregateDispatcherHeartbeats(needCompleteStatus boo Watermark: heartbeatpb.NewMaxWatermark(), RedoWatermark: heartbeatpb.NewMaxWatermark(), } - toCleanMap := make([]*cleanMap, 0) dispatcherCount := 0 diff --git a/heartbeatpb/heartbeat.pb.go b/heartbeatpb/heartbeat.pb.go index f2d2be03db..f757395370 100644 --- a/heartbeatpb/heartbeat.pb.go +++ b/heartbeatpb/heartbeat.pb.go @@ -148,6 +148,36 @@ func (NodeLiveness) EnumDescriptor() ([]byte, []int) { return fileDescriptor_6d584080fdadb670, []int{3} } +// NodeResourceUsageStatus distinguishes rolling-upgrade compatibility from a +// telemetry interruption on nodes that have declared support for reporting. +type NodeResourceUsageStatus int32 + +const ( + NodeResourceUsageStatus_UNSUPPORTED NodeResourceUsageStatus = 0 + NodeResourceUsageStatus_INCOMPLETE NodeResourceUsageStatus = 1 + NodeResourceUsageStatus_AVAILABLE NodeResourceUsageStatus = 2 +) + +var NodeResourceUsageStatus_name = map[int32]string{ + 0: "UNSUPPORTED", + 1: "INCOMPLETE", + 2: "AVAILABLE", +} + +var NodeResourceUsageStatus_value = map[string]int32{ + "UNSUPPORTED": 0, + "INCOMPLETE": 1, + "AVAILABLE": 2, +} + +func (x NodeResourceUsageStatus) String() string { + return proto.EnumName(NodeResourceUsageStatus_name, int32(x)) +} + +func (NodeResourceUsageStatus) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{4} +} + type BlockStage int32 const ( @@ -176,7 +206,7 @@ func (x BlockStage) String() string { } func (BlockStage) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{4} + return fileDescriptor_6d584080fdadb670, []int{5} } type InfluenceType int32 @@ -204,7 +234,7 @@ func (x InfluenceType) String() string { } func (InfluenceType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{5} + return fileDescriptor_6d584080fdadb670, []int{6} } // RouteTableAdmissionAction describes the name-level lifecycle change for one @@ -239,7 +269,7 @@ func (x RouteTableAdmissionAction) String() string { } func (RouteTableAdmissionAction) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{6} + return fileDescriptor_6d584080fdadb670, []int{7} } type ComponentState int32 @@ -279,7 +309,7 @@ func (x ComponentState) String() string { } func (ComponentState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{7} + return fileDescriptor_6d584080fdadb670, []int{8} } type ChecksumState int32 @@ -311,7 +341,7 @@ func (x ChecksumState) String() string { } func (ChecksumState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6d584080fdadb670, []int{8} + return fileDescriptor_6d584080fdadb670, []int{9} } type TableSpan struct { @@ -1623,11 +1653,13 @@ type NodeHeartbeat struct { NodeEpoch uint64 `protobuf:"varint,2,opt,name=node_epoch,json=nodeEpoch,proto3" json:"node_epoch,omitempty"` // dispatcher_drain_target_* reports the manager-level dispatcher drain target // currently applied on this node. Empty target means the drain target is clear. - DispatcherDrainTargetNodeId string `protobuf:"bytes,3,opt,name=dispatcher_drain_target_node_id,json=dispatcherDrainTargetNodeId,proto3" json:"dispatcher_drain_target_node_id,omitempty"` - DispatcherDrainTargetEpoch uint64 `protobuf:"varint,4,opt,name=dispatcher_drain_target_epoch,json=dispatcherDrainTargetEpoch,proto3" json:"dispatcher_drain_target_epoch,omitempty"` - WriteLeaseRequestSeq uint64 `protobuf:"varint,5,opt,name=write_lease_request_seq,json=writeLeaseRequestSeq,proto3" json:"write_lease_request_seq,omitempty"` - WriteLeaseProtocolVersion uint32 `protobuf:"varint,6,opt,name=write_lease_protocol_version,json=writeLeaseProtocolVersion,proto3" json:"write_lease_protocol_version,omitempty"` - WriteLeaseWitnessAck *WriteLeaseWitnessAck `protobuf:"bytes,7,opt,name=write_lease_witness_ack,json=writeLeaseWitnessAck,proto3" json:"write_lease_witness_ack,omitempty"` + DispatcherDrainTargetNodeId string `protobuf:"bytes,3,opt,name=dispatcher_drain_target_node_id,json=dispatcherDrainTargetNodeId,proto3" json:"dispatcher_drain_target_node_id,omitempty"` + DispatcherDrainTargetEpoch uint64 `protobuf:"varint,4,opt,name=dispatcher_drain_target_epoch,json=dispatcherDrainTargetEpoch,proto3" json:"dispatcher_drain_target_epoch,omitempty"` + WriteLeaseRequestSeq uint64 `protobuf:"varint,5,opt,name=write_lease_request_seq,json=writeLeaseRequestSeq,proto3" json:"write_lease_request_seq,omitempty"` + WriteLeaseProtocolVersion uint32 `protobuf:"varint,6,opt,name=write_lease_protocol_version,json=writeLeaseProtocolVersion,proto3" json:"write_lease_protocol_version,omitempty"` + WriteLeaseWitnessAck *WriteLeaseWitnessAck `protobuf:"bytes,7,opt,name=write_lease_witness_ack,json=writeLeaseWitnessAck,proto3" json:"write_lease_witness_ack,omitempty"` + NodeResourceUsage *NodeResourceUsage `protobuf:"bytes,8,opt,name=node_resource_usage,json=nodeResourceUsage,proto3" json:"node_resource_usage,omitempty"` + NodeResourceUsageProtocolVersion uint32 `protobuf:"varint,9,opt,name=node_resource_usage_protocol_version,json=nodeResourceUsageProtocolVersion,proto3" json:"node_resource_usage_protocol_version,omitempty"` } func (m *NodeHeartbeat) Reset() { *m = NodeHeartbeat{} } @@ -1712,6 +1744,20 @@ func (m *NodeHeartbeat) GetWriteLeaseWitnessAck() *WriteLeaseWitnessAck { return nil } +func (m *NodeHeartbeat) GetNodeResourceUsage() *NodeResourceUsage { + if m != nil { + return m.NodeResourceUsage + } + return nil +} + +func (m *NodeHeartbeat) GetNodeResourceUsageProtocolVersion() uint32 { + if m != nil { + return m.NodeResourceUsageProtocolVersion + } + return 0 +} + type WriteLeaseWitnessChallenge struct { CoordinatorVersion int64 `protobuf:"varint,1,opt,name=coordinator_version,json=coordinatorVersion,proto3" json:"coordinator_version,omitempty"` CoordinatorNodeEpoch uint64 `protobuf:"varint,2,opt,name=coordinator_node_epoch,json=coordinatorNodeEpoch,proto3" json:"coordinator_node_epoch,omitempty"` @@ -1865,11 +1911,13 @@ func (m *WriteLeaseWitnessAck) GetNonce() []byte { } type NodeHeartbeatResponse struct { - CoordinatorVersion int64 `protobuf:"varint,1,opt,name=coordinator_version,json=coordinatorVersion,proto3" json:"coordinator_version,omitempty"` - TargetNodeEpoch uint64 `protobuf:"varint,2,opt,name=target_node_epoch,json=targetNodeEpoch,proto3" json:"target_node_epoch,omitempty"` - RequestSeq uint64 `protobuf:"varint,3,opt,name=request_seq,json=requestSeq,proto3" json:"request_seq,omitempty"` - LeaseDurationMs uint64 `protobuf:"varint,4,opt,name=lease_duration_ms,json=leaseDurationMs,proto3" json:"lease_duration_ms,omitempty"` - WitnessChallenge *WriteLeaseWitnessChallenge `protobuf:"bytes,5,opt,name=witness_challenge,json=witnessChallenge,proto3" json:"witness_challenge,omitempty"` + CoordinatorVersion int64 `protobuf:"varint,1,opt,name=coordinator_version,json=coordinatorVersion,proto3" json:"coordinator_version,omitempty"` + TargetNodeEpoch uint64 `protobuf:"varint,2,opt,name=target_node_epoch,json=targetNodeEpoch,proto3" json:"target_node_epoch,omitempty"` + RequestSeq uint64 `protobuf:"varint,3,opt,name=request_seq,json=requestSeq,proto3" json:"request_seq,omitempty"` + LeaseDurationMs uint64 `protobuf:"varint,4,opt,name=lease_duration_ms,json=leaseDurationMs,proto3" json:"lease_duration_ms,omitempty"` + WitnessChallenge *WriteLeaseWitnessChallenge `protobuf:"bytes,5,opt,name=witness_challenge,json=witnessChallenge,proto3" json:"witness_challenge,omitempty"` + NodeResourceUsages []*NodeResourceUsage `protobuf:"bytes,6,rep,name=node_resource_usages,json=nodeResourceUsages,proto3" json:"node_resource_usages,omitempty"` + NodeResourceUsageStatus NodeResourceUsageStatus `protobuf:"varint,7,opt,name=node_resource_usage_status,json=nodeResourceUsageStatus,proto3,enum=heartbeatpb.NodeResourceUsageStatus" json:"node_resource_usage_status,omitempty"` } func (m *NodeHeartbeatResponse) Reset() { *m = NodeHeartbeatResponse{} } @@ -1940,6 +1988,20 @@ func (m *NodeHeartbeatResponse) GetWitnessChallenge() *WriteLeaseWitnessChalleng return nil } +func (m *NodeHeartbeatResponse) GetNodeResourceUsages() []*NodeResourceUsage { + if m != nil { + return m.NodeResourceUsages + } + return nil +} + +func (m *NodeHeartbeatResponse) GetNodeResourceUsageStatus() NodeResourceUsageStatus { + if m != nil { + return m.NodeResourceUsageStatus + } + return NodeResourceUsageStatus_UNSUPPORTED +} + // SetNodeLivenessRequest asks a node to transition its local liveness. type SetNodeLivenessRequest struct { Target NodeLiveness `protobuf:"varint,1,opt,name=target,proto3,enum=heartbeatpb.NodeLiveness" json:"target,omitempty"` @@ -4208,11 +4270,78 @@ func (m *DispatcherSetChecksumUpdateRequest) GetChecksum() *DispatcherSetChecksu return nil } +// NodeResourceUsage carries node-wide EventStore resource usage. Node heartbeat +// requests report the cumulative counter; coordinator responses set node_id and +// the derived rate for every entry in the cluster snapshot. +type NodeResourceUsage struct { + // Set by node heartbeat requests. + EventStoreWriteBytes uint64 `protobuf:"varint,1,opt,name=event_store_write_bytes,json=eventStoreWriteBytes,proto3" json:"event_store_write_bytes,omitempty"` + // Set by coordinator heartbeat responses. + NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + // Set by coordinator heartbeat responses. + EventStoreWriteBytesPerSecond uint64 `protobuf:"varint,3,opt,name=event_store_write_bytes_per_second,json=eventStoreWriteBytesPerSecond,proto3" json:"event_store_write_bytes_per_second,omitempty"` +} + +func (m *NodeResourceUsage) Reset() { *m = NodeResourceUsage{} } +func (m *NodeResourceUsage) String() string { return proto.CompactTextString(m) } +func (*NodeResourceUsage) ProtoMessage() {} +func (*NodeResourceUsage) Descriptor() ([]byte, []int) { + return fileDescriptor_6d584080fdadb670, []int{56} +} +func (m *NodeResourceUsage) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *NodeResourceUsage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_NodeResourceUsage.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *NodeResourceUsage) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeResourceUsage.Merge(m, src) +} +func (m *NodeResourceUsage) XXX_Size() int { + return m.Size() +} +func (m *NodeResourceUsage) XXX_DiscardUnknown() { + xxx_messageInfo_NodeResourceUsage.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeResourceUsage proto.InternalMessageInfo + +func (m *NodeResourceUsage) GetEventStoreWriteBytes() uint64 { + if m != nil { + return m.EventStoreWriteBytes + } + return 0 +} + +func (m *NodeResourceUsage) GetNodeId() string { + if m != nil { + return m.NodeId + } + return "" +} + +func (m *NodeResourceUsage) GetEventStoreWriteBytesPerSecond() uint64 { + if m != nil { + return m.EventStoreWriteBytesPerSecond + } + return 0 +} + func init() { proto.RegisterEnum("heartbeatpb.Action", Action_name, Action_value) proto.RegisterEnum("heartbeatpb.ScheduleAction", ScheduleAction_name, ScheduleAction_value) proto.RegisterEnum("heartbeatpb.OperatorType", OperatorType_name, OperatorType_value) proto.RegisterEnum("heartbeatpb.NodeLiveness", NodeLiveness_name, NodeLiveness_value) + proto.RegisterEnum("heartbeatpb.NodeResourceUsageStatus", NodeResourceUsageStatus_name, NodeResourceUsageStatus_value) proto.RegisterEnum("heartbeatpb.BlockStage", BlockStage_name, BlockStage_value) proto.RegisterEnum("heartbeatpb.InfluenceType", InfluenceType_name, InfluenceType_value) proto.RegisterEnum("heartbeatpb.RouteTableAdmissionAction", RouteTableAdmissionAction_name, RouteTableAdmissionAction_value) @@ -4274,218 +4403,231 @@ func init() { proto.RegisterType((*DispatcherSetChecksum)(nil), "heartbeatpb.DispatcherSetChecksum") proto.RegisterType((*DispatcherSetChecksumAckResponse)(nil), "heartbeatpb.DispatcherSetChecksumAckResponse") proto.RegisterType((*DispatcherSetChecksumUpdateRequest)(nil), "heartbeatpb.DispatcherSetChecksumUpdateRequest") + proto.RegisterType((*NodeResourceUsage)(nil), "heartbeatpb.NodeResourceUsage") } func init() { proto.RegisterFile("heartbeatpb/heartbeat.proto", fileDescriptor_6d584080fdadb670) } var fileDescriptor_6d584080fdadb670 = []byte{ - // 3285 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x1a, 0x4d, 0x73, 0x1c, 0x47, - 0xd5, 0x33, 0xfb, 0xa5, 0x7d, 0xd2, 0x4a, 0xe3, 0xb6, 0x2c, 0xcb, 0xb6, 0x2c, 0xcb, 0x93, 0x00, - 0xca, 0x26, 0xd8, 0xd8, 0x89, 0x29, 0x08, 0x21, 0x66, 0xbd, 0xab, 0xc4, 0x5b, 0xd6, 0x4a, 0xaa, - 0x59, 0x25, 0x4e, 0x85, 0xc3, 0x32, 0x9a, 0x69, 0xaf, 0x26, 0xda, 0x9d, 0x59, 0xcf, 0xcc, 0x5a, - 0xb6, 0xab, 0x80, 0x4a, 0x51, 0xdc, 0x38, 0xc0, 0x09, 0x0e, 0xe4, 0xc2, 0x89, 0x13, 0xc5, 0x1f, - 0xa0, 0xc8, 0x81, 0x03, 0x27, 0x2a, 0xc5, 0x81, 0x0a, 0x17, 0x48, 0x25, 0x47, 0x2e, 0x50, 0x54, - 0xc1, 0x95, 0xea, 0xaf, 0x99, 0x9e, 0x9d, 0x59, 0xad, 0x84, 0xb6, 0x52, 0x14, 0xa7, 0x99, 0x7e, - 0xfd, 0xde, 0xeb, 0xd7, 0xef, 0xbd, 0x7e, 0xfd, 0xfa, 0x75, 0xc3, 0xe5, 0x7d, 0x6c, 0xfa, 0xe1, - 0x1e, 0x36, 0xc3, 0xc1, 0xde, 0x8d, 0xe8, 0xff, 0xfa, 0xc0, 0xf7, 0x42, 0x0f, 0xcd, 0x4a, 0x9d, - 0xfa, 0x53, 0x28, 0xef, 0x9a, 0x7b, 0x3d, 0xdc, 0x1e, 0x98, 0x2e, 0x5a, 0x86, 0x12, 0x6d, 0x34, - 0x1b, 0xcb, 0xca, 0x9a, 0xb2, 0x9e, 0x33, 0x44, 0x13, 0x5d, 0x82, 0x99, 0x76, 0x68, 0xfa, 0xe1, - 0x7d, 0xfc, 0x74, 0x59, 0x5d, 0x53, 0xd6, 0xe7, 0x8c, 0xa8, 0x8d, 0x96, 0xa0, 0xb8, 0xe1, 0xda, - 0xa4, 0x27, 0x47, 0x7b, 0x78, 0x0b, 0xad, 0x02, 0xdc, 0xc7, 0x4f, 0x83, 0x81, 0x69, 0x11, 0x86, - 0xf9, 0x35, 0x65, 0xbd, 0x62, 0x48, 0x10, 0xfd, 0x4f, 0x2a, 0x68, 0xf7, 0x88, 0x28, 0x77, 0xb1, - 0x19, 0x1a, 0xf8, 0xd1, 0x10, 0x07, 0x21, 0xfa, 0x26, 0xcc, 0x59, 0xfb, 0xa6, 0xdb, 0xc5, 0x0f, - 0x31, 0xb6, 0xb9, 0x1c, 0xb3, 0xb7, 0x2e, 0x5e, 0x97, 0x64, 0xbe, 0x5e, 0x97, 0x10, 0x8c, 0x04, - 0x3a, 0x7a, 0x05, 0xca, 0x87, 0x66, 0x88, 0xfd, 0xbe, 0xe9, 0x1f, 0x50, 0x41, 0x67, 0x6f, 0x2d, - 0x25, 0x68, 0x1f, 0x88, 0x5e, 0x23, 0x46, 0x44, 0xaf, 0x41, 0xc5, 0xc7, 0xb6, 0x17, 0xf5, 0xd1, - 0x89, 0x8c, 0xa7, 0x4c, 0x22, 0xa3, 0xaf, 0xc1, 0x4c, 0x10, 0x9a, 0xe1, 0x30, 0xc0, 0xc1, 0x72, - 0x7e, 0x2d, 0xb7, 0x3e, 0x7b, 0x6b, 0x25, 0x41, 0x18, 0xe9, 0xb7, 0x4d, 0xb1, 0x8c, 0x08, 0x1b, - 0xad, 0xc3, 0x82, 0xe5, 0xf5, 0x07, 0xb8, 0x87, 0x43, 0xcc, 0x3a, 0x97, 0x0b, 0x6b, 0xca, 0xfa, - 0x8c, 0x31, 0x0a, 0x46, 0x2f, 0x42, 0x0e, 0xfb, 0xfe, 0x72, 0x31, 0x43, 0x1b, 0xc6, 0xd0, 0x75, - 0x1d, 0xb7, 0xbb, 0xe1, 0xfb, 0x9e, 0x6f, 0x10, 0x2c, 0xfd, 0x87, 0x0a, 0x94, 0x63, 0xf1, 0x74, - 0xa2, 0x51, 0x6c, 0x1d, 0x0c, 0x3c, 0xc7, 0x0d, 0x77, 0x03, 0xaa, 0xd1, 0xbc, 0x91, 0x80, 0x11, - 0x53, 0xf9, 0x38, 0xf0, 0x7a, 0x8f, 0xb1, 0xbd, 0x1b, 0x50, 0xbd, 0xe5, 0x0d, 0x09, 0x82, 0x34, - 0xc8, 0x05, 0xf8, 0x11, 0x55, 0x4b, 0xde, 0x20, 0xbf, 0x84, 0x6b, 0xcf, 0x0c, 0xc2, 0xf6, 0x53, - 0xd7, 0xa2, 0x34, 0x79, 0xc6, 0x55, 0x86, 0xe9, 0xdf, 0x05, 0xad, 0xe1, 0x04, 0x03, 0x33, 0xb4, - 0xf6, 0xb1, 0x5f, 0xb3, 0x42, 0xc7, 0x73, 0xd1, 0x8b, 0x50, 0x34, 0xe9, 0x1f, 0x95, 0x63, 0xfe, - 0xd6, 0xb9, 0xc4, 0x5c, 0x18, 0x92, 0xc1, 0x51, 0x88, 0xd7, 0xd5, 0xbd, 0x7e, 0xdf, 0x09, 0x23, - 0xa1, 0xa2, 0x36, 0x5a, 0x83, 0xd9, 0x66, 0x40, 0x86, 0xda, 0x21, 0x73, 0xa0, 0xa2, 0xcd, 0x18, - 0x32, 0x48, 0xaf, 0x43, 0xae, 0x56, 0xbf, 0x9f, 0x60, 0xa2, 0x1c, 0xcd, 0x44, 0x4d, 0x33, 0x31, - 0x00, 0x35, 0xbb, 0xae, 0xe7, 0x63, 0xfb, 0x6e, 0xcf, 0xb3, 0x0e, 0xb8, 0x39, 0x4e, 0xc7, 0xf3, - 0x07, 0x2a, 0x9c, 0x6f, 0xba, 0x0f, 0x7b, 0x43, 0x4c, 0x14, 0x15, 0xab, 0x28, 0x40, 0xdf, 0x82, - 0x4a, 0xd4, 0xb1, 0xfb, 0x74, 0x80, 0xb9, 0x92, 0x2e, 0x25, 0x94, 0x94, 0xc0, 0x30, 0x92, 0x04, - 0xe8, 0x0e, 0x54, 0x62, 0x86, 0xcd, 0x06, 0xd1, 0x5b, 0x2e, 0xe5, 0x32, 0x32, 0x86, 0x91, 0xc4, - 0xa7, 0x2b, 0xdd, 0xda, 0xc7, 0x7d, 0xb3, 0xd9, 0xa0, 0x4a, 0xcd, 0x19, 0x51, 0x1b, 0xdd, 0x87, - 0x73, 0xf8, 0x89, 0xd5, 0x1b, 0xda, 0x58, 0xa2, 0xb1, 0xa9, 0xed, 0x8f, 0x1c, 0x22, 0x8b, 0x4a, - 0xff, 0x99, 0x2a, 0xbb, 0x07, 0x57, 0xec, 0x3b, 0x70, 0xde, 0xc9, 0xd2, 0x0c, 0x8f, 0x03, 0x7a, - 0xb6, 0x22, 0x64, 0x4c, 0x23, 0x9b, 0x01, 0xba, 0x1d, 0x39, 0x1e, 0x0b, 0x0b, 0x57, 0xc6, 0x88, - 0x3b, 0xe2, 0x82, 0x3a, 0xe4, 0x4c, 0x4b, 0x04, 0x04, 0x2d, 0xe9, 0xac, 0xf5, 0xfb, 0x06, 0xe9, - 0x44, 0xdb, 0x80, 0x9c, 0x94, 0x8f, 0x70, 0xad, 0x5c, 0x4d, 0x4a, 0x9c, 0x42, 0x33, 0x32, 0x48, - 0xf5, 0x4f, 0x14, 0x38, 0x2b, 0x45, 0xc6, 0x60, 0xe0, 0xb9, 0x01, 0x3e, 0x6d, 0x68, 0x6c, 0x01, - 0xb2, 0x47, 0xd4, 0x8d, 0x85, 0x7b, 0x8c, 0x53, 0x86, 0x90, 0x31, 0x4d, 0x88, 0x10, 0xe4, 0xfb, - 0x9e, 0x8d, 0xb9, 0x8f, 0xd0, 0x7f, 0xf4, 0x02, 0x68, 0x7d, 0xd3, 0x71, 0x43, 0xd3, 0x71, 0xb1, - 0xdf, 0xc1, 0x03, 0xcf, 0xda, 0xe7, 0x81, 0x61, 0x21, 0x86, 0x6f, 0x10, 0xb0, 0xfe, 0x04, 0xce, - 0xd5, 0xa5, 0x08, 0xd4, 0xc2, 0x41, 0x60, 0x76, 0x4f, 0x3d, 0xc7, 0xd1, 0x58, 0xa7, 0xa6, 0x63, - 0x9d, 0xfe, 0x5b, 0x05, 0x16, 0x0c, 0x6c, 0x7b, 0x2d, 0x1c, 0x9a, 0x53, 0x1a, 0x76, 0x52, 0xf8, - 0x1c, 0x15, 0x2b, 0x97, 0x11, 0x82, 0x4f, 0xa0, 0xbb, 0xef, 0xc1, 0x15, 0x32, 0x01, 0x23, 0x1a, - 0x60, 0xc7, 0xf7, 0xba, 0x3e, 0x0e, 0x82, 0xcf, 0x67, 0x3a, 0xfa, 0x2f, 0x15, 0x58, 0x49, 0x0a, - 0xf0, 0x86, 0xe7, 0x1f, 0x9a, 0xbe, 0xfd, 0x39, 0xa9, 0x33, 0x4b, 0x55, 0xb9, 0x6c, 0x55, 0xfd, - 0x53, 0x91, 0x83, 0x4c, 0xdd, 0x73, 0x1f, 0x3a, 0x5d, 0x54, 0x85, 0x7c, 0x30, 0x30, 0x5d, 0x2e, - 0xd6, 0x52, 0xf6, 0x66, 0x6d, 0x50, 0x1c, 0x92, 0x12, 0x05, 0x24, 0xd1, 0x89, 0x04, 0x11, 0x4d, - 0x32, 0x49, 0x5b, 0x0a, 0x72, 0x3c, 0x44, 0x1c, 0x11, 0x05, 0x13, 0xe8, 0x24, 0xce, 0x06, 0x22, - 0xce, 0xe6, 0x59, 0x9c, 0x15, 0xed, 0x68, 0x6d, 0x15, 0xa4, 0xb5, 0x55, 0x05, 0x2d, 0x38, 0x70, - 0x06, 0x8d, 0xd6, 0x66, 0x2d, 0x68, 0x73, 0x89, 0x8a, 0x74, 0x6f, 0x49, 0xc1, 0xf5, 0x0f, 0x55, - 0xb8, 0x48, 0x82, 0xb6, 0x3d, 0xec, 0x49, 0x31, 0x77, 0x4a, 0x29, 0xd6, 0x6d, 0x28, 0x5a, 0x54, - 0x8f, 0x13, 0x02, 0x29, 0x53, 0xb6, 0xc1, 0x91, 0x51, 0x1d, 0xe6, 0x03, 0x2e, 0x12, 0x0b, 0xb1, - 0x54, 0x61, 0xf3, 0xb7, 0x2e, 0x27, 0xc8, 0xdb, 0x09, 0x14, 0x63, 0x84, 0x84, 0x88, 0xee, 0x0d, - 0xb0, 0x6f, 0x86, 0x9e, 0x4f, 0xb7, 0xc7, 0x3c, 0x65, 0x91, 0x14, 0x7d, 0x5b, 0x42, 0x30, 0x12, - 0xe8, 0x99, 0x8e, 0x53, 0xc8, 0x76, 0x9c, 0x5f, 0xa8, 0xb0, 0xd4, 0xc2, 0x7e, 0x77, 0xfa, 0xfa, - 0xbb, 0x03, 0x15, 0xfb, 0x84, 0x3b, 0x74, 0x02, 0x1f, 0x35, 0x01, 0xf5, 0x89, 0x64, 0x76, 0xe3, - 0x44, 0xee, 0x97, 0x41, 0x14, 0x39, 0x5a, 0x7e, 0x42, 0x10, 0x1f, 0xa3, 0xa4, 0x1d, 0x38, 0xd7, - 0x8a, 0x40, 0xf7, 0xc4, 0xc0, 0xe8, 0xeb, 0x52, 0x42, 0xac, 0x64, 0xec, 0x2f, 0x31, 0xcd, 0x68, - 0x46, 0xac, 0x7f, 0xac, 0x40, 0xa5, 0xe1, 0x9b, 0x8e, 0x2b, 0x42, 0x1a, 0x7a, 0x1e, 0xe6, 0x43, - 0xd3, 0xef, 0xe2, 0xb0, 0xe3, 0x7a, 0x36, 0xee, 0x38, 0x36, 0xd5, 0x77, 0xd9, 0x98, 0x63, 0xd0, - 0x2d, 0xcf, 0xc6, 0x4d, 0x1b, 0x5d, 0x03, 0xde, 0xe6, 0x02, 0xb3, 0xb5, 0x3a, 0xcb, 0x60, 0x54, - 0x58, 0xf4, 0x55, 0xb8, 0xc0, 0x51, 0x62, 0x75, 0x76, 0x2c, 0x6f, 0xc8, 0x93, 0xc7, 0x8a, 0x71, - 0x9e, 0x75, 0xcb, 0x1e, 0x3c, 0x74, 0x43, 0xf4, 0x06, 0xac, 0x71, 0x3a, 0x92, 0x58, 0x38, 0xdd, - 0xfd, 0xb0, 0x63, 0x13, 0x09, 0x3b, 0x7d, 0xef, 0x31, 0xe6, 0x0c, 0xd8, 0xe1, 0x66, 0x85, 0xe1, - 0x35, 0x39, 0x1a, 0x9d, 0x47, 0xcb, 0x7b, 0x8c, 0x29, 0x1f, 0xfd, 0x57, 0x39, 0xd0, 0x46, 0x67, - 0x7e, 0x5a, 0x5f, 0xba, 0x02, 0x40, 0xfe, 0x3a, 0x44, 0x7f, 0x98, 0x4e, 0xba, 0x6c, 0x94, 0x09, - 0x84, 0xb0, 0xc7, 0xe8, 0x26, 0x14, 0x58, 0x4f, 0xd6, 0x52, 0xab, 0x7b, 0xfd, 0x81, 0xe7, 0x62, - 0x37, 0xa4, 0xb8, 0x06, 0xc3, 0x44, 0xcf, 0x41, 0x25, 0xde, 0x96, 0x3a, 0x61, 0x94, 0xd8, 0x27, - 0xf6, 0x2a, 0x7e, 0x1a, 0x29, 0x64, 0x38, 0x6e, 0xea, 0x34, 0x82, 0xbe, 0x00, 0xf3, 0x7b, 0x9e, - 0x17, 0x06, 0xa1, 0x6f, 0x0e, 0x3a, 0xb6, 0xe7, 0x62, 0x1e, 0xb6, 0x2a, 0x11, 0xb4, 0xe1, 0xb9, - 0x38, 0x75, 0xa0, 0x28, 0xa5, 0x0f, 0x14, 0xa8, 0x06, 0xf3, 0x4c, 0xf5, 0x03, 0xee, 0x1d, 0xcb, - 0x33, 0x54, 0x5f, 0xc9, 0xfc, 0x38, 0xe1, 0x3f, 0x46, 0xc5, 0x4e, 0xb8, 0x53, 0x96, 0x77, 0x97, - 0xb3, 0xbd, 0xfb, 0xc3, 0x1c, 0x54, 0x88, 0x7b, 0xc5, 0x8e, 0x7d, 0x1b, 0x66, 0x7a, 0xce, 0x63, - 0xec, 0x92, 0x91, 0x95, 0x8c, 0xd0, 0x43, 0xb0, 0x37, 0x39, 0x82, 0x11, 0xa1, 0x12, 0x2b, 0x51, - 0xdf, 0x95, 0x5d, 0xb3, 0x4c, 0x20, 0xcc, 0x31, 0x1b, 0x70, 0x55, 0xf2, 0x48, 0x36, 0xc1, 0x11, - 0x97, 0xcf, 0x51, 0xcb, 0x5e, 0x8e, 0xd1, 0xe8, 0x1c, 0x77, 0xe5, 0x15, 0x50, 0x83, 0x2b, 0xe3, - 0xb8, 0xc8, 0xc9, 0xc4, 0xa5, 0x4c, 0x1e, 0x4c, 0x90, 0xdb, 0x70, 0xe1, 0xd0, 0x77, 0x42, 0xdc, - 0xe9, 0x61, 0x33, 0xc0, 0x1d, 0x9f, 0xc5, 0xbb, 0x0e, 0x39, 0xf9, 0xb1, 0x00, 0xb0, 0x48, 0xbb, - 0x37, 0x49, 0x2f, 0x0f, 0x86, 0x6d, 0xfc, 0x08, 0xdd, 0x81, 0x15, 0x99, 0x8c, 0x16, 0x19, 0x2c, - 0xaf, 0xd7, 0x79, 0x8c, 0xfd, 0x80, 0xc4, 0xf9, 0x22, 0x5d, 0x1c, 0x17, 0x63, 0xda, 0x1d, 0x8e, - 0xf1, 0x36, 0x43, 0x40, 0xef, 0x24, 0xc7, 0x3d, 0x74, 0x42, 0xa2, 0xb6, 0x0e, 0xc9, 0xbb, 0x4b, - 0xd4, 0xbe, 0xd7, 0x92, 0x07, 0xf1, 0x88, 0xd1, 0x03, 0x86, 0x59, 0xb3, 0x0e, 0x64, 0xd1, 0x62, - 0xa8, 0xfe, 0x0f, 0x05, 0x2e, 0xa5, 0xd0, 0xeb, 0xfb, 0x66, 0xaf, 0x87, 0xdd, 0x2e, 0x46, 0x37, - 0xe0, 0x9c, 0xe5, 0x79, 0xbe, 0xed, 0xb8, 0x64, 0x8b, 0x88, 0x04, 0x66, 0xb5, 0x0f, 0x24, 0x75, - 0x09, 0x49, 0x5f, 0x81, 0x25, 0x99, 0x20, 0x65, 0xd5, 0x45, 0xa9, 0x77, 0x2b, 0x32, 0xf0, 0x3a, - 0x68, 0x01, 0xee, 0x3d, 0x4c, 0x28, 0x94, 0xe5, 0x2b, 0xf3, 0x04, 0x2e, 0xa9, 0xf2, 0x25, 0x40, - 0x62, 0xf6, 0x12, 0x6f, 0x66, 0x39, 0x8d, 0xf7, 0xc4, 0x7c, 0x17, 0xa1, 0xe0, 0x7a, 0xae, 0xc5, - 0xf2, 0x84, 0x39, 0x83, 0x35, 0xf4, 0xbf, 0x29, 0xb0, 0x98, 0xa5, 0xa2, 0xff, 0xcf, 0xd9, 0xfe, - 0x54, 0x85, 0xf3, 0x89, 0x45, 0x1a, 0x1d, 0x97, 0x4e, 0x3c, 0xdd, 0x2a, 0x9c, 0x95, 0x97, 0x9d, - 0x3c, 0xd3, 0x85, 0x78, 0xb3, 0x61, 0xc2, 0x5c, 0x85, 0xd9, 0xf4, 0xfc, 0xc0, 0x8f, 0xe7, 0x56, - 0x85, 0xb3, 0xcc, 0x9b, 0xed, 0xa1, 0x6f, 0x92, 0xdc, 0xa5, 0xd3, 0x17, 0xb1, 0x74, 0x81, 0x76, - 0x34, 0x38, 0xbc, 0x15, 0xa0, 0x5d, 0x38, 0x2b, 0xf4, 0x60, 0x09, 0xdf, 0xa4, 0xb3, 0x9c, 0xbd, - 0xf5, 0xa5, 0xa3, 0x3d, 0x3f, 0x72, 0xe5, 0x48, 0x5f, 0x11, 0x44, 0x7f, 0x0f, 0x96, 0xda, 0x4c, - 0xe4, 0x28, 0x24, 0xf1, 0x04, 0xe6, 0x26, 0x14, 0xd9, 0x7c, 0x26, 0x07, 0x31, 0x8e, 0x38, 0x21, - 0x84, 0xe9, 0x7d, 0xb8, 0x90, 0x1a, 0x8b, 0x9b, 0xe1, 0x65, 0x28, 0x99, 0x83, 0x41, 0xcf, 0xc1, - 0xf6, 0xe4, 0xd1, 0x04, 0xe6, 0xa4, 0xe1, 0xde, 0x83, 0xab, 0x6d, 0x79, 0xa3, 0x96, 0x22, 0x99, - 0x98, 0xe3, 0xb4, 0xd2, 0x06, 0xfd, 0x09, 0x5c, 0xae, 0xc7, 0xbe, 0x72, 0x57, 0xec, 0x59, 0x62, - 0x9c, 0x65, 0x28, 0x25, 0x3d, 0x4b, 0x34, 0x27, 0x86, 0x45, 0x75, 0x42, 0x58, 0xd4, 0xff, 0xac, - 0xc2, 0x4a, 0xf6, 0xd0, 0x5c, 0xb5, 0xff, 0x7d, 0x9e, 0x45, 0x96, 0x76, 0xb4, 0x93, 0x66, 0x89, - 0xb5, 0x28, 0x76, 0xcd, 0x44, 0xa0, 0xfe, 0x9f, 0xd9, 0xa9, 0x26, 0xe9, 0xb6, 0x30, 0x49, 0xb7, - 0xff, 0x52, 0x60, 0xb1, 0x66, 0xdb, 0xb1, 0x86, 0x84, 0x3d, 0x5f, 0x00, 0x95, 0xfb, 0xca, 0x91, - 0x69, 0x98, 0xea, 0xd8, 0x68, 0x29, 0x71, 0x10, 0x9a, 0x8b, 0x4e, 0x3a, 0xa9, 0x14, 0x2a, 0xeb, - 0xb8, 0x5f, 0x85, 0xb3, 0x4e, 0xd0, 0x71, 0xf1, 0x61, 0x27, 0x4e, 0xe8, 0xe8, 0xc4, 0x67, 0x8c, - 0x05, 0x27, 0xd8, 0xc2, 0x87, 0xf1, 0x70, 0x24, 0xd8, 0x1c, 0xf0, 0xb2, 0x39, 0x51, 0x31, 0x9b, - 0x1c, 0x08, 0x50, 0xd3, 0xce, 0x4c, 0x6a, 0x8a, 0xd9, 0x49, 0xcd, 0xef, 0x14, 0xb8, 0x60, 0x60, - 0x92, 0xba, 0x9e, 0x6a, 0xee, 0xcb, 0x50, 0xb2, 0xcc, 0xc0, 0x32, 0x6d, 0xcc, 0x0b, 0x9c, 0xa2, - 0x49, 0x7a, 0x7c, 0xca, 0xdf, 0xe6, 0x35, 0x59, 0xd1, 0x1c, 0x9d, 0x46, 0xfe, 0x58, 0xd3, 0x18, - 0x73, 0xf2, 0xf8, 0x43, 0x0e, 0x2e, 0xc5, 0x13, 0x48, 0xad, 0xca, 0x53, 0xa6, 0xd5, 0xe3, 0x2c, - 0x7b, 0x91, 0x2e, 0x38, 0x5f, 0x32, 0x6a, 0x54, 0x0d, 0xb0, 0xe0, 0x5a, 0x68, 0xee, 0xf5, 0x70, - 0x27, 0xf4, 0x9d, 0x6e, 0x97, 0x88, 0xff, 0x18, 0xbb, 0x89, 0xa3, 0x86, 0x73, 0x8c, 0x42, 0xe9, - 0x15, 0xca, 0x63, 0x97, 0xb1, 0xd8, 0x20, 0x1c, 0xe4, 0x92, 0x69, 0xb6, 0xd3, 0x14, 0xb2, 0x9d, - 0xc6, 0x24, 0xc7, 0x16, 0x59, 0x20, 0x1f, 0xdb, 0xde, 0x88, 0x3c, 0xc5, 0x49, 0xf2, 0xac, 0xc8, - 0xf2, 0x18, 0xd8, 0xf6, 0x12, 0xe2, 0x8c, 0x18, 0xb4, 0x74, 0x2c, 0x83, 0xce, 0x64, 0x1b, 0xf4, - 0x8f, 0x39, 0xb8, 0x9c, 0x69, 0xd0, 0xe9, 0x14, 0x3f, 0x6f, 0x43, 0x21, 0x18, 0x98, 0xae, 0x38, - 0x6c, 0x27, 0xab, 0xb2, 0xd1, 0x68, 0x71, 0xf1, 0x87, 0x61, 0x8b, 0x83, 0x4e, 0xee, 0x38, 0xd7, - 0x2e, 0xc7, 0x3b, 0x3a, 0xbd, 0x04, 0x88, 0x1a, 0x22, 0x89, 0xc9, 0xbc, 0x5c, 0x23, 0x3d, 0x72, - 0x55, 0x14, 0x35, 0xa0, 0x2c, 0x0a, 0x18, 0xc1, 0x72, 0x91, 0x8a, 0xfe, 0xc5, 0xcc, 0x7a, 0x49, - 0xaa, 0x4a, 0x61, 0xc4, 0x84, 0x99, 0x66, 0x28, 0x65, 0x9a, 0x01, 0x6d, 0xc2, 0x02, 0x2d, 0x13, - 0x74, 0xe2, 0x61, 0x67, 0xe8, 0xb0, 0xcf, 0x25, 0x77, 0x96, 0xcc, 0xca, 0x88, 0x31, 0x4f, 0x69, - 0x45, 0x01, 0x26, 0xd0, 0xff, 0xa2, 0xc2, 0x6a, 0x6c, 0xd4, 0x1d, 0x2f, 0x08, 0xa7, 0xbd, 0x52, - 0x8f, 0xb5, 0xec, 0xd4, 0x53, 0x2e, 0xbb, 0x9b, 0x50, 0x62, 0xa5, 0x39, 0xb2, 0xea, 0x89, 0x32, - 0x2e, 0xa4, 0x6c, 0xd0, 0x37, 0x9b, 0xee, 0x43, 0xcf, 0x10, 0x78, 0xe8, 0x55, 0x98, 0xa3, 0x66, - 0x16, 0x74, 0xf9, 0xa3, 0xe9, 0x66, 0x09, 0x72, 0x9b, 0xd3, 0x9e, 0x20, 0x0c, 0x7e, 0xa0, 0xc2, - 0xd5, 0xb1, 0x0a, 0x9e, 0xce, 0xca, 0xf9, 0x5c, 0x34, 0x7c, 0xa2, 0x75, 0x76, 0xa2, 0x5b, 0x06, - 0x88, 0xb5, 0x9c, 0xb8, 0xda, 0x52, 0x46, 0xae, 0xb6, 0x56, 0x05, 0xe6, 0x96, 0xd9, 0x17, 0x95, - 0x14, 0x09, 0x82, 0xae, 0x43, 0x91, 0x46, 0x07, 0xe1, 0x02, 0x19, 0x55, 0x63, 0x6a, 0x49, 0x8e, - 0xa5, 0xd7, 0xf9, 0xbd, 0x3a, 0x1d, 0x78, 0xfc, 0xbd, 0xfa, 0x0a, 0x47, 0x93, 0x46, 0x8d, 0x01, - 0xfa, 0x6f, 0x54, 0x40, 0xe9, 0xe0, 0x44, 0xf6, 0xe9, 0x31, 0x76, 0x4c, 0xe8, 0x5c, 0xe5, 0xf7, - 0xf6, 0x62, 0xca, 0xea, 0xc8, 0x94, 0x45, 0x19, 0x3c, 0x77, 0x8c, 0x32, 0xf8, 0x1b, 0xa0, 0x59, - 0xa2, 0x5e, 0xd4, 0x09, 0xe2, 0x0b, 0xae, 0x09, 0x45, 0xa5, 0x05, 0x4b, 0x6e, 0x0f, 0x83, 0x74, - 0x8c, 0x2c, 0x64, 0xc4, 0xc8, 0x97, 0x61, 0x76, 0xaf, 0xe7, 0x59, 0x07, 0xbc, 0xac, 0xc5, 0x76, - 0x29, 0x94, 0x5c, 0x3b, 0x94, 0x3d, 0xec, 0x89, 0x4b, 0x33, 0x1c, 0x95, 0x32, 0x4b, 0x71, 0x29, - 0x53, 0xff, 0xb9, 0x02, 0x4b, 0xf1, 0xf2, 0xa8, 0xf7, 0xbc, 0xa8, 0x6e, 0x71, 0xda, 0x55, 0x21, - 0x65, 0x39, 0x6a, 0x32, 0xcb, 0x39, 0xc1, 0xe5, 0xc4, 0x07, 0x0a, 0x5c, 0x48, 0x89, 0x37, 0x9d, - 0x55, 0xbb, 0x0c, 0xa5, 0x60, 0x68, 0x59, 0x38, 0x08, 0x84, 0x7c, 0xbc, 0x79, 0x12, 0xf9, 0x7e, - 0xa4, 0x80, 0x16, 0xdf, 0xb1, 0x32, 0xc7, 0x9e, 0xc2, 0x15, 0xf5, 0x25, 0x98, 0xe1, 0xee, 0xcf, - 0xb6, 0xe3, 0x9c, 0x11, 0xb5, 0x8f, 0xba, 0x7d, 0xd6, 0xbf, 0x0d, 0x05, 0x8a, 0x37, 0xe1, 0x99, - 0xca, 0x38, 0x77, 0x5f, 0x81, 0x72, 0x7b, 0xd0, 0x73, 0x68, 0x20, 0xe2, 0xa9, 0x69, 0x0c, 0xd0, - 0xdf, 0x57, 0xe1, 0x9c, 0xe1, 0x0d, 0x43, 0x4c, 0x59, 0xd5, 0xec, 0xbe, 0x13, 0xf0, 0xa2, 0x80, - 0xd6, 0xf6, 0x86, 0xbe, 0x85, 0xa5, 0xe8, 0xc0, 0x4e, 0x92, 0x29, 0x38, 0x5a, 0x87, 0x05, 0x06, - 0x1b, 0x5d, 0xd2, 0xa3, 0x60, 0xc2, 0x95, 0x1d, 0x67, 0x24, 0xae, 0xec, 0xe4, 0x94, 0x82, 0x13, - 0xae, 0x0c, 0x16, 0x73, 0xcd, 0x33, 0xae, 0x23, 0x60, 0xf4, 0x3a, 0x14, 0xf9, 0xd5, 0x4a, 0x81, - 0xda, 0x24, 0x99, 0x2a, 0x64, 0xcc, 0x4e, 0xdc, 0x75, 0xb3, 0xaf, 0xee, 0xc2, 0xbc, 0xd0, 0x16, - 0x73, 0xad, 0x23, 0x34, 0xbd, 0x06, 0xb3, 0xdb, 0x3d, 0x7b, 0x44, 0xd9, 0x32, 0x88, 0x60, 0x6c, - 0xe1, 0xc3, 0x11, 0x6b, 0xca, 0x20, 0xfd, 0xdf, 0x39, 0x28, 0xb0, 0xc5, 0xbb, 0x02, 0xe5, 0x66, - 0x40, 0x6f, 0xc0, 0x79, 0x99, 0x60, 0xc6, 0x88, 0x01, 0x44, 0x0a, 0xfa, 0x1b, 0xdf, 0xc1, 0xf1, - 0x26, 0xba, 0x03, 0xb3, 0xec, 0x57, 0x84, 0xe6, 0xf4, 0x85, 0xd4, 0xa8, 0x03, 0x1b, 0x32, 0x05, - 0xba, 0x0f, 0x67, 0xb7, 0x30, 0xb6, 0x1b, 0xbe, 0x37, 0x18, 0x08, 0x0c, 0x9e, 0xa6, 0x4f, 0x60, - 0x93, 0xa6, 0x43, 0xaf, 0xc1, 0x02, 0x01, 0xd6, 0x6c, 0x3b, 0x62, 0xc5, 0x4a, 0xe4, 0x28, 0x1d, - 0x5b, 0x8d, 0x51, 0x54, 0x54, 0x87, 0xf9, 0xb7, 0x06, 0xb6, 0x19, 0x62, 0xae, 0x42, 0x91, 0xf0, - 0x5d, 0xce, 0x4a, 0x1a, 0xb8, 0x81, 0x8c, 0x11, 0x92, 0xd1, 0xc7, 0x27, 0xa5, 0xd4, 0xe3, 0x13, - 0xf4, 0x65, 0x7a, 0x27, 0xd0, 0xc5, 0x34, 0x11, 0x9f, 0x1f, 0x49, 0x49, 0xc4, 0x23, 0x84, 0x2e, - 0xbb, 0x0f, 0xe8, 0x62, 0xb4, 0x0b, 0x8b, 0x19, 0x8e, 0x13, 0x2c, 0x97, 0xa9, 0x6c, 0x6b, 0x93, - 0x3c, 0xcc, 0xc8, 0xa4, 0xd6, 0xbf, 0x0f, 0x8b, 0xd1, 0x0e, 0x23, 0xbf, 0xab, 0x39, 0xc1, 0xce, - 0xb6, 0x2e, 0xee, 0x36, 0xd4, 0xb1, 0xdb, 0x03, 0xbf, 0xd2, 0xc8, 0x78, 0xa9, 0xa0, 0xff, 0x5d, - 0x21, 0xab, 0x2a, 0xf1, 0x2e, 0xeb, 0x24, 0x83, 0x67, 0x6d, 0x87, 0xea, 0x34, 0xb6, 0xc3, 0xac, - 0x52, 0xc1, 0x4d, 0x38, 0xcf, 0x72, 0xae, 0xc0, 0x79, 0x86, 0x3b, 0x03, 0xec, 0x77, 0x02, 0x6c, - 0x79, 0x2e, 0x3b, 0x4e, 0xaa, 0x06, 0xa2, 0x9d, 0x6d, 0xe7, 0x19, 0xde, 0xc1, 0x7e, 0x9b, 0xf6, - 0x64, 0x5d, 0x20, 0xeb, 0xbf, 0x56, 0x00, 0xc9, 0x0f, 0x4f, 0xa6, 0xb3, 0x11, 0xbe, 0x09, 0x95, - 0xbd, 0x98, 0x69, 0xf4, 0xa0, 0xe4, 0x5a, 0x76, 0x36, 0x21, 0x8f, 0x9f, 0xa4, 0xcb, 0xb4, 0x92, - 0x0d, 0x73, 0x72, 0xfa, 0x47, 0x70, 0x42, 0x27, 0x0a, 0xc0, 0xf4, 0x9f, 0xc0, 0x5c, 0xcf, 0x16, - 0x91, 0x96, 0xfe, 0x13, 0x98, 0x25, 0x78, 0x95, 0x0d, 0xfa, 0x4f, 0x82, 0x48, 0x9f, 0x3d, 0x4f, - 0xe0, 0xe1, 0x53, 0x34, 0xf5, 0x57, 0x60, 0x6e, 0xf4, 0x52, 0x74, 0xdf, 0xe9, 0xee, 0xf3, 0x87, - 0x5d, 0xf4, 0x1f, 0x69, 0x90, 0xeb, 0x79, 0x87, 0x3c, 0xfc, 0x90, 0x5f, 0x22, 0x9b, 0xac, 0x96, - 0xe3, 0x51, 0x51, 0x69, 0xe3, 0x60, 0x4f, 0xff, 0xc9, 0xa6, 0x25, 0xce, 0xcc, 0x5c, 0xb4, 0xa8, - 0xad, 0x7f, 0x07, 0xae, 0x6e, 0x7a, 0x5d, 0xa9, 0x0a, 0x18, 0xbf, 0xb9, 0x98, 0x8e, 0x01, 0xf5, - 0xf7, 0x15, 0x58, 0x1b, 0x3f, 0xc4, 0x74, 0xb2, 0x91, 0x49, 0x0f, 0x4a, 0x7a, 0x44, 0x97, 0xd8, - 0x3a, 0x08, 0x86, 0xfd, 0x16, 0x0e, 0x4d, 0xf4, 0x15, 0xb1, 0xb6, 0xb3, 0x72, 0x0b, 0x81, 0x99, - 0x58, 0xe3, 0x55, 0xd0, 0x2c, 0x19, 0xde, 0xc6, 0x8f, 0xf8, 0x38, 0x29, 0xb8, 0xfe, 0x13, 0x05, - 0xce, 0x4b, 0x4f, 0x9c, 0x70, 0x28, 0x38, 0xa2, 0x45, 0x28, 0xb0, 0xfb, 0x5c, 0x66, 0x44, 0xd6, - 0x20, 0x9e, 0xf3, 0xc4, 0xf3, 0xef, 0x11, 0xe3, 0xf2, 0xed, 0x87, 0x37, 0xd1, 0x12, 0x14, 0x9f, - 0x78, 0xfe, 0xa6, 0x77, 0xc8, 0xd7, 0x2d, 0x6f, 0xb1, 0xec, 0xab, 0x4f, 0x29, 0xf2, 0xbc, 0x4c, - 0xc4, 0x9a, 0x84, 0x22, 0x18, 0xf6, 0x09, 0x05, 0x4b, 0x7c, 0x79, 0x8b, 0xa4, 0x82, 0x6b, 0x99, - 0x32, 0xd5, 0xac, 0x83, 0x69, 0x59, 0x61, 0x11, 0x0a, 0x72, 0x95, 0x9b, 0x35, 0x32, 0xdf, 0x71, - 0xf1, 0xe7, 0x9e, 0xf9, 0xe8, 0xb9, 0xa7, 0xfe, 0x57, 0x05, 0xf4, 0x4c, 0xf9, 0xd8, 0xfe, 0x33, - 0xa5, 0x60, 0x72, 0x0a, 0x09, 0xd1, 0xeb, 0x30, 0x23, 0x2c, 0xcd, 0xef, 0x4e, 0xf4, 0x71, 0x8f, - 0xda, 0x62, 0xe9, 0x8d, 0x88, 0xa6, 0x7a, 0x45, 0x24, 0x4f, 0xa8, 0x0c, 0x05, 0x7a, 0xd1, 0xa2, - 0x9d, 0x41, 0x33, 0x90, 0xdf, 0x31, 0x83, 0x40, 0x53, 0xaa, 0xeb, 0x2c, 0x37, 0x92, 0xde, 0xa2, - 0x00, 0x14, 0xeb, 0x3e, 0x36, 0x29, 0x1e, 0x40, 0x91, 0x15, 0x55, 0x35, 0xa5, 0xda, 0x82, 0x39, - 0xf9, 0x09, 0x0a, 0x61, 0xb7, 0xdd, 0xa9, 0xd9, 0xb6, 0x76, 0x06, 0xcd, 0xc1, 0xcc, 0x76, 0x47, - 0x20, 0x12, 0xa2, 0xed, 0x4e, 0x8b, 0xfc, 0xab, 0x68, 0x16, 0x4a, 0xdb, 0x1d, 0x9a, 0x8d, 0x6a, - 0x39, 0xd6, 0xa0, 0x25, 0x16, 0x2d, 0x5f, 0xbd, 0x0d, 0x73, 0xf2, 0x1d, 0x09, 0x61, 0x57, 0xdb, - 0x6c, 0xbe, 0xbd, 0xc1, 0xd8, 0x35, 0x8c, 0x5a, 0x73, 0xab, 0xb9, 0xf5, 0xa6, 0xa6, 0x90, 0x56, - 0x7b, 0x77, 0x7b, 0x67, 0x87, 0xb4, 0xd4, 0xea, 0xab, 0x00, 0xf1, 0x66, 0x4e, 0xe6, 0xb1, 0xb5, - 0xbd, 0x45, 0x68, 0x66, 0xa1, 0xf4, 0xa0, 0xd6, 0xdc, 0x65, 0x24, 0xa4, 0x61, 0xb0, 0x86, 0x4a, - 0x70, 0x1a, 0x04, 0x27, 0x57, 0x7d, 0x69, 0x24, 0xc5, 0x47, 0x25, 0xc8, 0xd5, 0x7a, 0x3d, 0xed, - 0x0c, 0x2a, 0x82, 0xda, 0xb8, 0xcb, 0x44, 0xdf, 0xf2, 0xfc, 0xbe, 0xd9, 0xd3, 0xd4, 0xea, 0x9b, - 0x70, 0x71, 0x6c, 0x6a, 0x49, 0xa5, 0x6d, 0xb4, 0x9a, 0xbb, 0x6c, 0x64, 0x63, 0x63, 0x73, 0xa3, - 0xd6, 0xde, 0xd0, 0x14, 0x84, 0x60, 0x9e, 0x37, 0x3a, 0xed, 0xfa, 0xbd, 0x8d, 0x56, 0x4d, 0x53, - 0xab, 0xcf, 0x60, 0x3e, 0xb9, 0x5f, 0x52, 0xf9, 0x3c, 0xff, 0xc0, 0x71, 0xbb, 0x8c, 0xbe, 0x1d, - 0xd2, 0x74, 0x8b, 0x49, 0xce, 0xf4, 0x68, 0x6b, 0x2a, 0xd2, 0x60, 0xae, 0xe9, 0x3a, 0xa1, 0x63, - 0xf6, 0x9c, 0x67, 0x04, 0x37, 0x87, 0x2a, 0x50, 0xde, 0xf1, 0xf1, 0xc0, 0xf4, 0x49, 0x33, 0x8f, - 0xe6, 0x01, 0xa8, 0x3a, 0x0d, 0x6c, 0xda, 0x4f, 0xb5, 0x02, 0x21, 0x78, 0x60, 0x3a, 0xa1, 0xe3, - 0x76, 0x99, 0x96, 0x8b, 0xd5, 0x6f, 0x40, 0x25, 0x11, 0x57, 0xd0, 0x59, 0xa8, 0xbc, 0xb5, 0xd5, - 0xdc, 0x6a, 0xee, 0x36, 0x6b, 0x9b, 0xcd, 0x77, 0x37, 0x1a, 0x4c, 0xdd, 0xad, 0x66, 0xbb, 0x55, - 0xdb, 0xad, 0xdf, 0xd3, 0x14, 0x32, 0x33, 0xf6, 0xab, 0xde, 0x7d, 0xfd, 0xf7, 0x9f, 0xae, 0x2a, - 0x1f, 0x7d, 0xba, 0xaa, 0x7c, 0xf2, 0xe9, 0xaa, 0xf2, 0xe3, 0xcf, 0x56, 0xcf, 0x7c, 0xf4, 0xd9, - 0xea, 0x99, 0x8f, 0x3f, 0x5b, 0x3d, 0xf3, 0xee, 0xf3, 0x5d, 0x27, 0xdc, 0x1f, 0xee, 0x5d, 0xb7, - 0xbc, 0xfe, 0x8d, 0x81, 0xe3, 0x76, 0x2d, 0x73, 0x70, 0x23, 0x74, 0x2c, 0xdb, 0xba, 0x21, 0xb9, - 0xe6, 0x5e, 0x91, 0xde, 0x5f, 0xbc, 0xfc, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x40, 0x67, 0x9a, - 0x8d, 0xb6, 0x2f, 0x00, 0x00, + // 3479 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x3a, 0x4d, 0x6f, 0x1c, 0xc7, + 0xb1, 0x9a, 0xd9, 0x2f, 0x6e, 0xf1, 0x6b, 0xd8, 0xa2, 0x48, 0x4a, 0xa2, 0x28, 0x6a, 0xac, 0xf7, + 0x1e, 0xbd, 0xf6, 0x93, 0x9e, 0x64, 0xeb, 0xe1, 0x3d, 0xc7, 0xb1, 0xb2, 0xda, 0xa5, 0xad, 0x85, + 0xb8, 0x4b, 0x62, 0x96, 0xb2, 0x0c, 0xe7, 0xb0, 0x19, 0xce, 0xb4, 0x96, 0x63, 0xee, 0xce, 0xac, + 0x66, 0x66, 0x45, 0x49, 0x40, 0x12, 0x18, 0x41, 0x6e, 0x39, 0x24, 0xb7, 0x1c, 0xe2, 0x4b, 0x4e, + 0x39, 0x19, 0xf9, 0x03, 0x41, 0x72, 0xc8, 0x21, 0x87, 0x20, 0x30, 0x72, 0x08, 0x9c, 0x4b, 0x62, + 0xd8, 0xc7, 0x5c, 0x12, 0x04, 0x48, 0xae, 0x41, 0x7f, 0xcd, 0xf4, 0xec, 0xcc, 0x72, 0xc9, 0x90, + 0x30, 0x82, 0x9c, 0x66, 0xba, 0xba, 0xaa, 0xba, 0xba, 0xaa, 0xba, 0xba, 0xba, 0xba, 0xe1, 0xf2, + 0x3e, 0x36, 0xfd, 0x70, 0x0f, 0x9b, 0xe1, 0x60, 0xef, 0x66, 0xf4, 0x7f, 0x63, 0xe0, 0x7b, 0xa1, + 0x87, 0xa6, 0xa5, 0x4e, 0xfd, 0x39, 0x94, 0x77, 0xcd, 0xbd, 0x1e, 0x6e, 0x0f, 0x4c, 0x17, 0xad, + 0x40, 0x89, 0x36, 0x1a, 0xf5, 0x15, 0x65, 0x5d, 0xd9, 0xc8, 0x19, 0xa2, 0x89, 0x2e, 0xc1, 0x54, + 0x3b, 0x34, 0xfd, 0xf0, 0x01, 0x7e, 0xbe, 0xa2, 0xae, 0x2b, 0x1b, 0x33, 0x46, 0xd4, 0x46, 0x4b, + 0x50, 0xdc, 0x74, 0x6d, 0xd2, 0x93, 0xa3, 0x3d, 0xbc, 0x85, 0xd6, 0x00, 0x1e, 0xe0, 0xe7, 0xc1, + 0xc0, 0xb4, 0x08, 0xc3, 0xfc, 0xba, 0xb2, 0x31, 0x6b, 0x48, 0x10, 0xfd, 0x77, 0x2a, 0x68, 0xf7, + 0x89, 0x28, 0xf7, 0xb0, 0x19, 0x1a, 0xf8, 0xc9, 0x10, 0x07, 0x21, 0xfa, 0x2a, 0xcc, 0x58, 0xfb, + 0xa6, 0xdb, 0xc5, 0x8f, 0x31, 0xb6, 0xb9, 0x1c, 0xd3, 0xb7, 0x2f, 0xde, 0x90, 0x64, 0xbe, 0x51, + 0x93, 0x10, 0x8c, 0x04, 0x3a, 0x7a, 0x1d, 0xca, 0x87, 0x66, 0x88, 0xfd, 0xbe, 0xe9, 0x1f, 0x50, + 0x41, 0xa7, 0x6f, 0x2f, 0x25, 0x68, 0x1f, 0x89, 0x5e, 0x23, 0x46, 0x44, 0x6f, 0xc2, 0xac, 0x8f, + 0x6d, 0x2f, 0xea, 0xa3, 0x13, 0x19, 0x4f, 0x99, 0x44, 0x46, 0xff, 0x07, 0x53, 0x41, 0x68, 0x86, + 0xc3, 0x00, 0x07, 0x2b, 0xf9, 0xf5, 0xdc, 0xc6, 0xf4, 0xed, 0xd5, 0x04, 0x61, 0xa4, 0xdf, 0x36, + 0xc5, 0x32, 0x22, 0x6c, 0xb4, 0x01, 0xf3, 0x96, 0xd7, 0x1f, 0xe0, 0x1e, 0x0e, 0x31, 0xeb, 0x5c, + 0x29, 0xac, 0x2b, 0x1b, 0x53, 0xc6, 0x28, 0x18, 0xbd, 0x02, 0x39, 0xec, 0xfb, 0x2b, 0xc5, 0x0c, + 0x6d, 0x18, 0x43, 0xd7, 0x75, 0xdc, 0xee, 0xa6, 0xef, 0x7b, 0xbe, 0x41, 0xb0, 0xf4, 0xef, 0x2a, + 0x50, 0x8e, 0xc5, 0xd3, 0x89, 0x46, 0xb1, 0x75, 0x30, 0xf0, 0x1c, 0x37, 0xdc, 0x0d, 0xa8, 0x46, + 0xf3, 0x46, 0x02, 0x46, 0x4c, 0xe5, 0xe3, 0xc0, 0xeb, 0x3d, 0xc5, 0xf6, 0x6e, 0x40, 0xf5, 0x96, + 0x37, 0x24, 0x08, 0xd2, 0x20, 0x17, 0xe0, 0x27, 0x54, 0x2d, 0x79, 0x83, 0xfc, 0x12, 0xae, 0x3d, + 0x33, 0x08, 0xdb, 0xcf, 0x5d, 0x8b, 0xd2, 0xe4, 0x19, 0x57, 0x19, 0xa6, 0x7f, 0x13, 0xb4, 0xba, + 0x13, 0x0c, 0xcc, 0xd0, 0xda, 0xc7, 0x7e, 0xd5, 0x0a, 0x1d, 0xcf, 0x45, 0xaf, 0x40, 0xd1, 0xa4, + 0x7f, 0x54, 0x8e, 0xb9, 0xdb, 0xe7, 0x13, 0x73, 0x61, 0x48, 0x06, 0x47, 0x21, 0x5e, 0x57, 0xf3, + 0xfa, 0x7d, 0x27, 0x8c, 0x84, 0x8a, 0xda, 0x68, 0x1d, 0xa6, 0x1b, 0x01, 0x19, 0x6a, 0x87, 0xcc, + 0x81, 0x8a, 0x36, 0x65, 0xc8, 0x20, 0xbd, 0x06, 0xb9, 0x6a, 0xed, 0x41, 0x82, 0x89, 0x72, 0x34, + 0x13, 0x35, 0xcd, 0xc4, 0x00, 0xd4, 0xe8, 0xba, 0x9e, 0x8f, 0xed, 0x7b, 0x3d, 0xcf, 0x3a, 0xe0, + 0xe6, 0x38, 0x1d, 0xcf, 0xef, 0xa8, 0x70, 0xa1, 0xe1, 0x3e, 0xee, 0x0d, 0x31, 0x51, 0x54, 0xac, + 0xa2, 0x00, 0x7d, 0x0d, 0x66, 0xa3, 0x8e, 0xdd, 0xe7, 0x03, 0xcc, 0x95, 0x74, 0x29, 0xa1, 0xa4, + 0x04, 0x86, 0x91, 0x24, 0x40, 0x77, 0x61, 0x36, 0x66, 0xd8, 0xa8, 0x13, 0xbd, 0xe5, 0x52, 0x2e, + 0x23, 0x63, 0x18, 0x49, 0x7c, 0xba, 0xd2, 0xad, 0x7d, 0xdc, 0x37, 0x1b, 0x75, 0xaa, 0xd4, 0x9c, + 0x11, 0xb5, 0xd1, 0x03, 0x38, 0x8f, 0x9f, 0x59, 0xbd, 0xa1, 0x8d, 0x25, 0x1a, 0x9b, 0xda, 0xfe, + 0xc8, 0x21, 0xb2, 0xa8, 0xf4, 0x1f, 0xaa, 0xb2, 0x7b, 0x70, 0xc5, 0xbe, 0x07, 0x17, 0x9c, 0x2c, + 0xcd, 0xf0, 0x38, 0xa0, 0x67, 0x2b, 0x42, 0xc6, 0x34, 0xb2, 0x19, 0xa0, 0x3b, 0x91, 0xe3, 0xb1, + 0xb0, 0x70, 0x65, 0x8c, 0xb8, 0x23, 0x2e, 0xa8, 0x43, 0xce, 0xb4, 0x44, 0x40, 0xd0, 0x92, 0xce, + 0x5a, 0x7b, 0x60, 0x90, 0x4e, 0xb4, 0x0d, 0xc8, 0x49, 0xf9, 0x08, 0xd7, 0xca, 0xd5, 0xa4, 0xc4, + 0x29, 0x34, 0x23, 0x83, 0x54, 0xff, 0x4c, 0x81, 0x05, 0x29, 0x32, 0x06, 0x03, 0xcf, 0x0d, 0xf0, + 0x69, 0x43, 0x63, 0x13, 0x90, 0x3d, 0xa2, 0x6e, 0x2c, 0xdc, 0x63, 0x9c, 0x32, 0x84, 0x8c, 0x69, + 0x42, 0x84, 0x20, 0xdf, 0xf7, 0x6c, 0xcc, 0x7d, 0x84, 0xfe, 0xa3, 0x97, 0x41, 0xeb, 0x9b, 0x8e, + 0x1b, 0x9a, 0x8e, 0x8b, 0xfd, 0x0e, 0x1e, 0x78, 0xd6, 0x3e, 0x0f, 0x0c, 0xf3, 0x31, 0x7c, 0x93, + 0x80, 0xf5, 0x67, 0x70, 0xbe, 0x26, 0x45, 0xa0, 0x26, 0x0e, 0x02, 0xb3, 0x7b, 0xea, 0x39, 0x8e, + 0xc6, 0x3a, 0x35, 0x1d, 0xeb, 0xf4, 0x9f, 0x2b, 0x30, 0x6f, 0x60, 0xdb, 0x6b, 0xe2, 0xd0, 0x3c, + 0xa3, 0x61, 0x27, 0x85, 0xcf, 0x51, 0xb1, 0x72, 0x19, 0x21, 0xf8, 0x04, 0xba, 0xfb, 0x16, 0x5c, + 0x21, 0x13, 0x30, 0xa2, 0x01, 0x76, 0x7c, 0xaf, 0xeb, 0xe3, 0x20, 0xf8, 0x72, 0xa6, 0xa3, 0xff, + 0x44, 0x81, 0xd5, 0xa4, 0x00, 0x6f, 0x7b, 0xfe, 0xa1, 0xe9, 0xdb, 0x5f, 0x92, 0x3a, 0xb3, 0x54, + 0x95, 0xcb, 0x56, 0xd5, 0x5f, 0x15, 0x39, 0xc8, 0xd4, 0x3c, 0xf7, 0xb1, 0xd3, 0x45, 0x15, 0xc8, + 0x07, 0x03, 0xd3, 0xe5, 0x62, 0x2d, 0x65, 0x6f, 0xd6, 0x06, 0xc5, 0x21, 0x29, 0x51, 0x40, 0x12, + 0x9d, 0x48, 0x10, 0xd1, 0x24, 0x93, 0xb4, 0xa5, 0x20, 0xc7, 0x43, 0xc4, 0x11, 0x51, 0x30, 0x81, + 0x4e, 0xe2, 0x6c, 0x20, 0xe2, 0x6c, 0x9e, 0xc5, 0x59, 0xd1, 0x8e, 0xd6, 0x56, 0x41, 0x5a, 0x5b, + 0x15, 0xd0, 0x82, 0x03, 0x67, 0x50, 0x6f, 0x6e, 0x55, 0x83, 0x36, 0x97, 0xa8, 0x48, 0xf7, 0x96, + 0x14, 0x5c, 0xff, 0x85, 0x0a, 0x17, 0x49, 0xd0, 0xb6, 0x87, 0x3d, 0x29, 0xe6, 0x9e, 0x51, 0x8a, + 0x75, 0x07, 0x8a, 0x16, 0xd5, 0xe3, 0x84, 0x40, 0xca, 0x94, 0x6d, 0x70, 0x64, 0x54, 0x83, 0xb9, + 0x80, 0x8b, 0xc4, 0x42, 0x2c, 0x55, 0xd8, 0xdc, 0xed, 0xcb, 0x09, 0xf2, 0x76, 0x02, 0xc5, 0x18, + 0x21, 0x21, 0xa2, 0x7b, 0x03, 0xec, 0x9b, 0xa1, 0xe7, 0xd3, 0xed, 0x31, 0x4f, 0x59, 0x24, 0x45, + 0xdf, 0x96, 0x10, 0x8c, 0x04, 0x7a, 0xa6, 0xe3, 0x14, 0xb2, 0x1d, 0xe7, 0xc7, 0x2a, 0x2c, 0x35, + 0xb1, 0xdf, 0x3d, 0x7b, 0xfd, 0xdd, 0x85, 0x59, 0xfb, 0x84, 0x3b, 0x74, 0x02, 0x1f, 0x35, 0x00, + 0xf5, 0x89, 0x64, 0x76, 0xfd, 0x44, 0xee, 0x97, 0x41, 0x14, 0x39, 0x5a, 0x7e, 0x42, 0x10, 0x1f, + 0xa3, 0xa4, 0x1d, 0x38, 0xdf, 0x8c, 0x40, 0xf7, 0xc5, 0xc0, 0xe8, 0xff, 0xa5, 0x84, 0x58, 0xc9, + 0xd8, 0x5f, 0x62, 0x9a, 0xd1, 0x8c, 0x58, 0xff, 0x54, 0x81, 0xd9, 0xba, 0x6f, 0x3a, 0xae, 0x08, + 0x69, 0xe8, 0x3a, 0xcc, 0x85, 0xa6, 0xdf, 0xc5, 0x61, 0xc7, 0xf5, 0x6c, 0xdc, 0x71, 0x6c, 0xaa, + 0xef, 0xb2, 0x31, 0xc3, 0xa0, 0x2d, 0xcf, 0xc6, 0x0d, 0x1b, 0x5d, 0x03, 0xde, 0xe6, 0x02, 0xb3, + 0xb5, 0x3a, 0xcd, 0x60, 0x54, 0x58, 0xf4, 0xbf, 0xb0, 0xcc, 0x51, 0x62, 0x75, 0x76, 0x2c, 0x6f, + 0xc8, 0x93, 0xc7, 0x59, 0xe3, 0x02, 0xeb, 0x96, 0x3d, 0x78, 0xe8, 0x86, 0xe8, 0x6d, 0x58, 0xe7, + 0x74, 0x24, 0xb1, 0x70, 0xba, 0xfb, 0x61, 0xc7, 0x26, 0x12, 0x76, 0xfa, 0xde, 0x53, 0xcc, 0x19, + 0xb0, 0xc3, 0xcd, 0x2a, 0xc3, 0x6b, 0x70, 0x34, 0x3a, 0x8f, 0xa6, 0xf7, 0x14, 0x53, 0x3e, 0xfa, + 0xc7, 0x39, 0xd0, 0x46, 0x67, 0x7e, 0x5a, 0x5f, 0xba, 0x02, 0x40, 0xfe, 0x3a, 0x44, 0x7f, 0x98, + 0x4e, 0xba, 0x6c, 0x94, 0x09, 0x84, 0xb0, 0xc7, 0xe8, 0x16, 0x14, 0x58, 0x4f, 0xd6, 0x52, 0xab, + 0x79, 0xfd, 0x81, 0xe7, 0x62, 0x37, 0xa4, 0xb8, 0x06, 0xc3, 0x44, 0x2f, 0xc1, 0x6c, 0xbc, 0x2d, + 0x75, 0xc2, 0x28, 0xb1, 0x4f, 0xec, 0x55, 0xfc, 0x34, 0x52, 0xc8, 0x70, 0xdc, 0xd4, 0x69, 0x04, + 0xfd, 0x07, 0xcc, 0xed, 0x79, 0x5e, 0x18, 0x84, 0xbe, 0x39, 0xe8, 0xd8, 0x9e, 0x8b, 0x79, 0xd8, + 0x9a, 0x8d, 0xa0, 0x75, 0xcf, 0xc5, 0xa9, 0x03, 0x45, 0x29, 0x7d, 0xa0, 0x40, 0x55, 0x98, 0x63, + 0xaa, 0x1f, 0x70, 0xef, 0x58, 0x99, 0xa2, 0xfa, 0x4a, 0xe6, 0xc7, 0x09, 0xff, 0x31, 0x66, 0xed, + 0x84, 0x3b, 0x65, 0x79, 0x77, 0x39, 0xdb, 0xbb, 0x3f, 0xcb, 0xc3, 0x2c, 0x71, 0xaf, 0xd8, 0xb1, + 0xef, 0xc0, 0x54, 0xcf, 0x79, 0x8a, 0x5d, 0x32, 0xb2, 0x92, 0x11, 0x7a, 0x08, 0xf6, 0x16, 0x47, + 0x30, 0x22, 0x54, 0x62, 0x25, 0xea, 0xbb, 0xb2, 0x6b, 0x96, 0x09, 0x84, 0x39, 0x66, 0x1d, 0xae, + 0x4a, 0x1e, 0xc9, 0x26, 0x38, 0xe2, 0xf2, 0x39, 0x6a, 0xd9, 0xcb, 0x31, 0x1a, 0x9d, 0xe3, 0xae, + 0xbc, 0x02, 0xaa, 0x70, 0x65, 0x1c, 0x17, 0x39, 0x99, 0xb8, 0x94, 0xc9, 0x83, 0x09, 0x72, 0x07, + 0x96, 0x0f, 0x7d, 0x27, 0xc4, 0x9d, 0x1e, 0x36, 0x03, 0xdc, 0xf1, 0x59, 0xbc, 0xeb, 0x90, 0x93, + 0x1f, 0x0b, 0x00, 0x8b, 0xb4, 0x7b, 0x8b, 0xf4, 0xf2, 0x60, 0xd8, 0xc6, 0x4f, 0xd0, 0x5d, 0x58, + 0x95, 0xc9, 0x68, 0x91, 0xc1, 0xf2, 0x7a, 0x9d, 0xa7, 0xd8, 0x0f, 0x48, 0x9c, 0x2f, 0xd2, 0xc5, + 0x71, 0x31, 0xa6, 0xdd, 0xe1, 0x18, 0xef, 0x32, 0x04, 0xf4, 0x5e, 0x72, 0xdc, 0x43, 0x27, 0x24, + 0x6a, 0xeb, 0x90, 0xbc, 0xbb, 0x44, 0xed, 0x7b, 0x2d, 0x79, 0x10, 0x8f, 0x18, 0x3d, 0x62, 0x98, + 0x55, 0xeb, 0x40, 0x16, 0x2d, 0x86, 0xa2, 0x16, 0x9c, 0xa7, 0x2a, 0x24, 0xc9, 0xc3, 0xd0, 0xb7, + 0x70, 0x67, 0x48, 0xf2, 0x13, 0xee, 0x35, 0x6b, 0x29, 0xdb, 0x19, 0x1c, 0xed, 0x21, 0xc1, 0x32, + 0x16, 0xdc, 0x51, 0x10, 0x6a, 0xc1, 0xf5, 0x0c, 0x7e, 0xe9, 0x29, 0x97, 0xe9, 0x94, 0xd7, 0x53, + 0x0c, 0x46, 0x66, 0xae, 0xff, 0x45, 0x81, 0x4b, 0xa9, 0xe9, 0xd4, 0xf6, 0xcd, 0x5e, 0x0f, 0xbb, + 0x5d, 0x8c, 0x6e, 0xc2, 0x79, 0xcb, 0xf3, 0x7c, 0xdb, 0x71, 0xc9, 0x16, 0x16, 0x71, 0x67, 0xb5, + 0x19, 0x24, 0x75, 0x09, 0x4d, 0xbe, 0x0e, 0x4b, 0x32, 0x41, 0xca, 0xeb, 0x16, 0xa5, 0xde, 0x56, + 0xe4, 0x80, 0x1b, 0xa0, 0x05, 0xb8, 0xf7, 0x38, 0x61, 0x70, 0x96, 0x4f, 0xcd, 0x11, 0xb8, 0x64, + 0xea, 0x57, 0x01, 0x09, 0xeb, 0x48, 0xbc, 0x99, 0x67, 0x69, 0xbc, 0x27, 0xe6, 0xbb, 0x08, 0x05, + 0xd7, 0x73, 0x2d, 0x96, 0xc7, 0xcc, 0x18, 0xac, 0xa1, 0xff, 0x49, 0x81, 0xc5, 0x2c, 0x13, 0xfe, + 0x7b, 0xce, 0xf6, 0xd7, 0x39, 0xb8, 0x90, 0x08, 0x22, 0xd1, 0x71, 0xee, 0xc4, 0xd3, 0xad, 0xc0, + 0x82, 0x1c, 0x16, 0xe4, 0x99, 0xce, 0xc7, 0x9b, 0x21, 0x13, 0xe6, 0x2a, 0x4c, 0xa7, 0xe7, 0x07, + 0x7e, 0x3c, 0xb7, 0x0a, 0x2c, 0xb0, 0xd5, 0x66, 0x0f, 0x7d, 0x93, 0xe4, 0x56, 0x9d, 0xbe, 0x88, + 0xf5, 0xf3, 0xb4, 0xa3, 0xce, 0xe1, 0xcd, 0x00, 0xed, 0xc2, 0x82, 0xd0, 0x83, 0x25, 0x7c, 0x93, + 0xce, 0x72, 0xfa, 0xf6, 0x7f, 0x1d, 0xbd, 0x32, 0x23, 0x57, 0x8e, 0xf4, 0x15, 0x3b, 0xf7, 0x0e, + 0x2c, 0x66, 0xac, 0x25, 0x92, 0xd4, 0xe6, 0x8e, 0xb1, 0x38, 0x51, 0x6a, 0x6d, 0x05, 0xc8, 0x84, + 0x4b, 0x59, 0xab, 0x93, 0x25, 0x17, 0x34, 0x94, 0xcc, 0xdd, 0xbe, 0x7e, 0x34, 0x5f, 0x9e, 0x90, + 0x2c, 0xbb, 0xd9, 0x1d, 0xfa, 0x07, 0xb0, 0xd4, 0x66, 0x7a, 0x8e, 0xe2, 0x3c, 0xcf, 0x0a, 0x6f, + 0x41, 0x91, 0x19, 0x61, 0xf2, 0xce, 0xc0, 0x11, 0x27, 0xec, 0x0b, 0x7a, 0x1f, 0x96, 0x53, 0x63, + 0x71, 0xdf, 0x79, 0x0d, 0x4a, 0xe6, 0x60, 0xd0, 0x73, 0xb0, 0x3d, 0x79, 0x34, 0x81, 0x39, 0x69, + 0xb8, 0x0f, 0xe0, 0x6a, 0x5b, 0xce, 0x7e, 0xa4, 0xed, 0x41, 0xcc, 0xf1, 0xac, 0x72, 0x31, 0xfd, + 0x19, 0x5c, 0xae, 0xc5, 0x0e, 0x7e, 0x4f, 0x24, 0x02, 0x62, 0x9c, 0x15, 0x28, 0x25, 0x97, 0x83, + 0x68, 0x4e, 0xdc, 0x6b, 0xd4, 0x09, 0x7b, 0x8d, 0xfe, 0x7b, 0x15, 0x56, 0xb3, 0x87, 0xe6, 0xaa, + 0xfd, 0xe7, 0x93, 0x57, 0x12, 0x8f, 0xa2, 0xf4, 0x24, 0x4b, 0xac, 0x45, 0x91, 0x8a, 0x24, 0x76, + 0xbf, 0x7f, 0x99, 0xed, 0x7f, 0x92, 0x6e, 0x0b, 0x93, 0x74, 0xfb, 0x37, 0x05, 0x16, 0xab, 0xb6, + 0x1d, 0x6b, 0x48, 0xd8, 0xf3, 0x65, 0x50, 0xb9, 0xaf, 0x1c, 0x99, 0xdb, 0xaa, 0x8e, 0x8d, 0x96, + 0x12, 0xa7, 0xcb, 0x99, 0xe8, 0xf8, 0x98, 0xca, 0x4b, 0xb3, 0x6a, 0x28, 0x15, 0x58, 0x70, 0x82, + 0x8e, 0x8b, 0x0f, 0x3b, 0x71, 0x96, 0x4c, 0x27, 0x3e, 0x65, 0xcc, 0x3b, 0x41, 0x0b, 0x1f, 0xc6, + 0xc3, 0x91, 0x08, 0x79, 0xc0, 0xef, 0x22, 0x88, 0x8a, 0xd9, 0xe4, 0x40, 0x80, 0x1a, 0x76, 0x66, + 0xa6, 0x58, 0xcc, 0xce, 0x14, 0x7f, 0xa9, 0xc0, 0xb2, 0x81, 0xc9, 0x79, 0xe0, 0x54, 0x73, 0x5f, + 0x81, 0x92, 0x65, 0x06, 0x96, 0x69, 0x63, 0x5e, 0x35, 0x16, 0x4d, 0xd2, 0xe3, 0x53, 0xfe, 0x36, + 0x2f, 0x74, 0x8b, 0xe6, 0xe8, 0x34, 0xf2, 0xc7, 0x9a, 0xc6, 0x98, 0xe3, 0xdc, 0x6f, 0x72, 0x70, + 0x29, 0x9e, 0x40, 0x6a, 0x55, 0x9e, 0xf2, 0xac, 0x32, 0xce, 0xb2, 0x17, 0xe9, 0x82, 0xf3, 0x25, + 0xa3, 0x46, 0x25, 0x16, 0x0b, 0xae, 0x85, 0xe6, 0x5e, 0x0f, 0x77, 0x42, 0xdf, 0xe9, 0x76, 0x89, + 0xf8, 0x4f, 0xb1, 0x9b, 0x38, 0xbf, 0x39, 0xc7, 0xa8, 0x3e, 0x5f, 0xa1, 0x3c, 0x76, 0x19, 0x8b, + 0x4d, 0xc2, 0x41, 0xae, 0x43, 0x67, 0x3b, 0x4d, 0x21, 0xdb, 0x69, 0x4c, 0x72, 0x16, 0x94, 0x05, + 0xf2, 0xb1, 0xed, 0x8d, 0xc8, 0x53, 0x9c, 0x24, 0xcf, 0xaa, 0x2c, 0x8f, 0x81, 0x6d, 0x2f, 0x21, + 0xce, 0x88, 0x41, 0x4b, 0xc7, 0x32, 0xe8, 0x54, 0xb6, 0x41, 0x7f, 0x9b, 0x83, 0xcb, 0x99, 0x06, + 0x3d, 0x9b, 0x8a, 0xf2, 0x1d, 0x28, 0x04, 0x03, 0xd3, 0x15, 0x15, 0x8c, 0x64, 0xa9, 0x3b, 0x1a, + 0x2d, 0xae, 0xa8, 0x31, 0x6c, 0x71, 0x7a, 0xcc, 0x1d, 0xe7, 0x2e, 0xeb, 0x78, 0xe7, 0xd1, 0x57, + 0x01, 0x51, 0x43, 0x24, 0x31, 0x99, 0x97, 0x6b, 0xa4, 0x47, 0x2e, 0x35, 0xa3, 0x3a, 0x94, 0x45, + 0x55, 0x48, 0x64, 0x1b, 0xff, 0x99, 0x59, 0x84, 0x4a, 0x95, 0x7e, 0x8c, 0x98, 0x30, 0xd3, 0x0c, + 0xa5, 0x4c, 0x33, 0xa0, 0x2d, 0x98, 0xa7, 0xb5, 0x97, 0x4e, 0x3c, 0xec, 0x14, 0x1d, 0xf6, 0xa5, + 0xe4, 0xce, 0x92, 0x59, 0x6e, 0x32, 0xe6, 0x28, 0xad, 0xa8, 0x6a, 0x05, 0xfa, 0x1f, 0x54, 0x58, + 0x8b, 0x8d, 0xba, 0xe3, 0x05, 0xe1, 0x59, 0xaf, 0xd4, 0x63, 0x2d, 0x3b, 0xf5, 0x94, 0xcb, 0xee, + 0x16, 0x94, 0x58, 0xbd, 0x93, 0xac, 0x7a, 0xa2, 0x8c, 0xe5, 0x94, 0x0d, 0xfa, 0x66, 0xc3, 0x7d, + 0xec, 0x19, 0x02, 0x0f, 0xbd, 0x01, 0x33, 0xd4, 0xcc, 0x82, 0x2e, 0x7f, 0x34, 0xdd, 0x34, 0x41, + 0x6e, 0x73, 0xda, 0x13, 0x84, 0xc1, 0x8f, 0x54, 0xb8, 0x3a, 0x56, 0xc1, 0x67, 0xb3, 0x72, 0xbe, + 0x14, 0x0d, 0x9f, 0x68, 0x9d, 0x9d, 0xe8, 0xea, 0x06, 0x62, 0x2d, 0x27, 0xee, 0x0b, 0x95, 0x91, + 0xfb, 0xc2, 0x35, 0x81, 0xd9, 0x32, 0xfb, 0xa2, 0x3c, 0x25, 0x41, 0xd0, 0x0d, 0x28, 0xd2, 0xe8, + 0x20, 0x5c, 0x20, 0xa3, 0x14, 0x4f, 0x2d, 0xc9, 0xb1, 0xf4, 0x1a, 0x7f, 0xac, 0x40, 0x07, 0x1e, + 0xff, 0x58, 0x61, 0x95, 0xa3, 0x49, 0xa3, 0xc6, 0x00, 0xfd, 0x67, 0x2a, 0xa0, 0x74, 0x70, 0x22, + 0xfb, 0xf4, 0x18, 0x3b, 0x26, 0x74, 0xae, 0xf2, 0xc7, 0x10, 0x62, 0xca, 0xea, 0xc8, 0x94, 0xc5, + 0xdd, 0x42, 0xee, 0x18, 0x77, 0x0b, 0x6f, 0x83, 0x66, 0x89, 0x22, 0x9c, 0x38, 0xa5, 0xe4, 0x27, + 0x57, 0xea, 0xe6, 0x2d, 0xb9, 0x3d, 0x0c, 0xd2, 0x31, 0xb2, 0x90, 0x11, 0x23, 0x5f, 0x83, 0xe9, + 0xbd, 0x9e, 0x67, 0x1d, 0xf0, 0x5a, 0x21, 0xdb, 0xa5, 0x50, 0x72, 0xed, 0x50, 0xf6, 0xb0, 0x27, + 0x6e, 0x22, 0x71, 0x54, 0x1f, 0x2e, 0xc5, 0xf5, 0x61, 0xfd, 0x47, 0x0a, 0x2c, 0xc5, 0xcb, 0xa3, + 0xd6, 0xf3, 0xa2, 0x62, 0xd0, 0x69, 0x57, 0x85, 0x94, 0xe5, 0xa8, 0xc9, 0x2c, 0xe7, 0x04, 0x37, + 0x3e, 0x1f, 0x29, 0xb0, 0x9c, 0x12, 0xef, 0x6c, 0x56, 0xed, 0x0a, 0x94, 0x82, 0xa1, 0x65, 0xe1, + 0x20, 0x10, 0xf2, 0xf1, 0xe6, 0x49, 0xe4, 0xfb, 0x9e, 0x02, 0x5a, 0x7c, 0x71, 0xcd, 0x1c, 0xfb, + 0x0c, 0xee, 0xfd, 0x2f, 0xc1, 0x14, 0x77, 0x7f, 0xb6, 0x1d, 0xe7, 0x8c, 0xa8, 0x7d, 0xd4, 0x95, + 0xbe, 0xfe, 0x75, 0x28, 0x50, 0xbc, 0x09, 0x6f, 0x7f, 0xc6, 0xb9, 0xfb, 0x2a, 0x94, 0xdb, 0x83, + 0x9e, 0x43, 0x03, 0x11, 0x4f, 0x4d, 0x63, 0x80, 0xfe, 0xa1, 0x0a, 0xe7, 0x0d, 0x6f, 0x18, 0x62, + 0xca, 0xaa, 0x6a, 0xf7, 0x9d, 0x80, 0x57, 0x32, 0xb4, 0x36, 0x3d, 0x5a, 0x4b, 0xd1, 0x81, 0x9d, + 0x24, 0x53, 0x70, 0xb4, 0x01, 0xf3, 0x0c, 0x36, 0xba, 0xa4, 0x47, 0xc1, 0x84, 0x2b, 0x3b, 0xce, + 0x48, 0x5c, 0xd9, 0xc9, 0x29, 0x05, 0x27, 0x5c, 0x19, 0x2c, 0xe6, 0x9a, 0x67, 0x5c, 0x47, 0xc0, + 0xe8, 0x2d, 0x28, 0xf2, 0xfb, 0xaa, 0x02, 0xb5, 0x49, 0x32, 0x55, 0xc8, 0x98, 0x9d, 0x78, 0x40, + 0xc0, 0xbe, 0xba, 0x0b, 0x73, 0x42, 0x5b, 0xcc, 0xb5, 0x8e, 0xd0, 0xf4, 0x3a, 0x4c, 0x6f, 0xf7, + 0xec, 0x11, 0x65, 0xcb, 0x20, 0x82, 0xd1, 0xc2, 0x87, 0x23, 0xd6, 0x94, 0x41, 0xfa, 0xdf, 0x73, + 0x50, 0x60, 0x8b, 0x77, 0x15, 0xca, 0x8d, 0x80, 0x3e, 0x2b, 0xe0, 0x65, 0x82, 0x29, 0x23, 0x06, + 0x10, 0x29, 0xe8, 0x6f, 0x7c, 0xb1, 0xc9, 0x9b, 0xe8, 0x2e, 0x4c, 0xb3, 0x5f, 0x11, 0x9a, 0xd3, + 0xb7, 0x7c, 0xa3, 0x0e, 0x6c, 0xc8, 0x14, 0xe8, 0x01, 0x2c, 0xb4, 0x30, 0xb6, 0xeb, 0xbe, 0x37, + 0x18, 0x08, 0x0c, 0x9e, 0xa6, 0x4f, 0x60, 0x93, 0xa6, 0x43, 0x6f, 0xc2, 0x3c, 0x01, 0x56, 0x6d, + 0x3b, 0x62, 0xc5, 0xee, 0x1d, 0x50, 0x3a, 0xb6, 0x1a, 0xa3, 0xa8, 0xa8, 0x06, 0x73, 0x0f, 0x07, + 0xb6, 0x19, 0x62, 0xae, 0x42, 0x91, 0xf0, 0x5d, 0xce, 0x4a, 0x1a, 0xb8, 0x81, 0x8c, 0x11, 0x92, + 0xd1, 0x17, 0x3d, 0xa5, 0xd4, 0x8b, 0x1e, 0xf4, 0xdf, 0xf4, 0xa2, 0x85, 0x57, 0x96, 0xe7, 0x46, + 0x52, 0x12, 0xf1, 0xb2, 0xa3, 0xcb, 0x2e, 0x59, 0xba, 0x18, 0xed, 0xc2, 0x62, 0x86, 0xe3, 0x04, + 0x2b, 0x65, 0x2a, 0xdb, 0xfa, 0x24, 0x0f, 0x33, 0x32, 0xa9, 0xf5, 0x6f, 0xc3, 0x62, 0xb4, 0xc3, + 0xc8, 0x8f, 0x95, 0x4e, 0xb0, 0xb3, 0x6d, 0x88, 0x0b, 0x23, 0x75, 0xec, 0xf6, 0xc0, 0xef, 0x89, + 0x32, 0x9e, 0x7f, 0xe8, 0x7f, 0x56, 0xc8, 0xaa, 0x4a, 0x3c, 0x76, 0x3b, 0xc9, 0xe0, 0x59, 0xdb, + 0xa1, 0x7a, 0x16, 0xdb, 0x61, 0x56, 0xa9, 0xe0, 0x16, 0x5c, 0x60, 0x39, 0x57, 0xe0, 0xbc, 0xc0, + 0x9d, 0x01, 0xf6, 0x3b, 0x01, 0xb6, 0x3c, 0x97, 0x1d, 0x27, 0x55, 0x03, 0xd1, 0xce, 0xb6, 0xf3, + 0x02, 0xef, 0x60, 0xbf, 0x4d, 0x7b, 0xb2, 0x6e, 0xe5, 0xf5, 0x9f, 0x2a, 0x80, 0xe4, 0xd7, 0x3c, + 0x67, 0xb3, 0x11, 0xbe, 0x03, 0xb3, 0x7b, 0x31, 0xd3, 0xe8, 0x95, 0xce, 0xb5, 0xec, 0x6c, 0x42, + 0x1e, 0x3f, 0x49, 0x97, 0x69, 0x25, 0x1b, 0x66, 0xe4, 0xf4, 0x8f, 0xe0, 0x84, 0x4e, 0x14, 0x80, + 0xe9, 0x3f, 0x81, 0xb9, 0x9e, 0x2d, 0x22, 0x2d, 0xfd, 0x27, 0x30, 0x4b, 0xf0, 0x2a, 0x1b, 0xf4, + 0x9f, 0x04, 0x91, 0x3e, 0x7b, 0xf3, 0xc1, 0xc3, 0xa7, 0x68, 0xea, 0xaf, 0xc3, 0xcc, 0xe8, 0x4d, + 0xf3, 0xbe, 0xd3, 0xdd, 0xe7, 0xaf, 0xe5, 0xe8, 0x3f, 0xd2, 0x20, 0xd7, 0xf3, 0x0e, 0x79, 0xf8, + 0x21, 0xbf, 0x44, 0x36, 0x59, 0x2d, 0xc7, 0xa3, 0xa2, 0xd2, 0xc6, 0xc1, 0x9e, 0xfe, 0x93, 0x4d, + 0x4b, 0x9c, 0x99, 0xb9, 0x68, 0x51, 0x5b, 0xff, 0x06, 0x5c, 0xdd, 0xf2, 0xba, 0x52, 0x15, 0x30, + 0x7e, 0xc8, 0x72, 0x36, 0x06, 0xd4, 0x3f, 0x54, 0x60, 0x7d, 0xfc, 0x10, 0x67, 0x93, 0x8d, 0x4c, + 0x7a, 0xa5, 0xd3, 0x23, 0xba, 0xc4, 0xd6, 0x41, 0x30, 0xec, 0x37, 0x71, 0x68, 0xa2, 0xff, 0x11, + 0x6b, 0x3b, 0x2b, 0xb7, 0x10, 0x98, 0x89, 0x35, 0x5e, 0x01, 0xcd, 0x92, 0xe1, 0x6d, 0xfc, 0x84, + 0x8f, 0x93, 0x82, 0xeb, 0x3f, 0x50, 0xe0, 0x82, 0xf4, 0x6e, 0x0c, 0x87, 0x82, 0x23, 0x5a, 0x84, + 0x02, 0xbb, 0x24, 0x67, 0x46, 0x64, 0x0d, 0xe2, 0x39, 0xcf, 0x3c, 0xff, 0x3e, 0x31, 0x2e, 0xdf, + 0x7e, 0x78, 0x13, 0x2d, 0x41, 0xf1, 0x99, 0xe7, 0x6f, 0x79, 0x87, 0x7c, 0xdd, 0xf2, 0x16, 0xcb, + 0xbe, 0xfa, 0x94, 0x22, 0xcf, 0xcb, 0x44, 0xac, 0x49, 0x28, 0x82, 0x61, 0x9f, 0x50, 0xb0, 0xc4, + 0x97, 0xb7, 0x48, 0x2a, 0xb8, 0x9e, 0x29, 0x53, 0xd5, 0x3a, 0x38, 0x2b, 0x2b, 0x2c, 0x42, 0x41, + 0xae, 0x72, 0xb3, 0x46, 0xe6, 0xe3, 0x38, 0xfe, 0x86, 0x36, 0x1f, 0xbd, 0xa1, 0xd5, 0xff, 0xa8, + 0x80, 0x9e, 0x29, 0x1f, 0xdb, 0x7f, 0xce, 0x28, 0x98, 0x9c, 0x42, 0x42, 0xf4, 0x16, 0x4c, 0x09, + 0x4b, 0xf3, 0x0b, 0x1f, 0x7d, 0xdc, 0x4b, 0xc1, 0x58, 0x7a, 0x23, 0xa2, 0xd1, 0x3f, 0x56, 0x60, + 0x21, 0x75, 0xc7, 0x82, 0xee, 0xc0, 0x32, 0x8f, 0xbd, 0xa1, 0xe7, 0xe3, 0x0e, 0x2b, 0x3a, 0xef, + 0x3d, 0x0f, 0xb1, 0x78, 0x4c, 0xbb, 0xc8, 0xa2, 0x2f, 0xe9, 0xa5, 0x57, 0x4a, 0xf7, 0x48, 0x1f, + 0x5a, 0x86, 0x92, 0x28, 0x88, 0xb3, 0xb8, 0x54, 0x74, 0x59, 0xed, 0xbb, 0x01, 0xfa, 0x18, 0x7e, + 0x72, 0x60, 0x67, 0xde, 0x74, 0x25, 0x8b, 0x75, 0x14, 0xe3, 0x2b, 0x57, 0x44, 0xb6, 0x87, 0xca, + 0x50, 0xa0, 0x08, 0xda, 0x39, 0x34, 0x05, 0xf9, 0x1d, 0x33, 0x08, 0x34, 0xa5, 0xb2, 0xc1, 0x92, + 0x39, 0xe9, 0x45, 0x12, 0x40, 0xb1, 0xe6, 0x63, 0x93, 0xe2, 0x01, 0x14, 0x59, 0x15, 0x58, 0x53, + 0x2a, 0x4d, 0x98, 0x91, 0x1f, 0x22, 0x11, 0x76, 0xdb, 0x9d, 0xaa, 0x6d, 0x6b, 0xe7, 0xd0, 0x0c, + 0x4c, 0x6d, 0x77, 0x04, 0x22, 0x21, 0xda, 0xee, 0x34, 0xc9, 0xbf, 0x8a, 0xa6, 0xa1, 0xb4, 0xdd, + 0xa1, 0xe9, 0xb3, 0x96, 0x63, 0x0d, 0x5a, 0x13, 0xd2, 0xf2, 0x95, 0x3b, 0x30, 0x23, 0x5f, 0xea, + 0x10, 0x76, 0xd5, 0xad, 0xc6, 0xbb, 0x9b, 0x8c, 0x5d, 0xdd, 0xa8, 0x36, 0x5a, 0x8d, 0xd6, 0x3b, + 0x9a, 0x42, 0x5a, 0xed, 0xdd, 0xed, 0x9d, 0x1d, 0xd2, 0x52, 0x2b, 0x0d, 0x58, 0x1e, 0x73, 0xc5, + 0x85, 0xe6, 0x61, 0xfa, 0x61, 0xab, 0xfd, 0x70, 0x67, 0x67, 0xdb, 0xd8, 0xdd, 0xac, 0x6b, 0xe7, + 0xd0, 0x1c, 0x40, 0xa3, 0x55, 0xdb, 0x6e, 0xee, 0x6c, 0x6d, 0xee, 0x6e, 0x6a, 0x0a, 0x9a, 0x85, + 0x72, 0xf5, 0xdd, 0x6a, 0x63, 0xab, 0x7a, 0x6f, 0x6b, 0x53, 0x53, 0x2b, 0x6f, 0x00, 0xc4, 0x89, + 0x0c, 0x51, 0x49, 0x6b, 0xbb, 0x45, 0x86, 0x9f, 0x86, 0xd2, 0xa3, 0x6a, 0x63, 0x97, 0x8d, 0x4e, + 0x1a, 0x06, 0x6b, 0xa8, 0x04, 0xa7, 0x4e, 0x70, 0x72, 0x95, 0x57, 0x47, 0x8e, 0x37, 0xa8, 0x04, + 0xb9, 0x6a, 0xaf, 0xa7, 0x9d, 0x43, 0x45, 0x50, 0xeb, 0xf7, 0x98, 0x16, 0x5a, 0x9e, 0xdf, 0x37, + 0x7b, 0x9a, 0x5a, 0x79, 0x07, 0x2e, 0x8e, 0x4d, 0xab, 0xe9, 0xc4, 0xeb, 0xcd, 0xc6, 0x2e, 0x1b, + 0xd9, 0xd8, 0xdc, 0xda, 0xac, 0xb6, 0x89, 0xb4, 0x08, 0xe6, 0x78, 0xa3, 0xd3, 0xae, 0xdd, 0xdf, + 0x6c, 0x56, 0x35, 0xb5, 0xf2, 0x02, 0xe6, 0x92, 0xb9, 0x02, 0x95, 0xcf, 0xf3, 0x0f, 0x1c, 0xb7, + 0xcb, 0xe8, 0xdb, 0x21, 0x4d, 0x35, 0x99, 0xe4, 0xcc, 0x24, 0xb6, 0xa6, 0x22, 0x0d, 0x66, 0x1a, + 0xae, 0x13, 0x3a, 0x66, 0xcf, 0x79, 0x41, 0x70, 0x73, 0x44, 0x19, 0x3b, 0x3e, 0x1e, 0x98, 0x3e, + 0x69, 0xe6, 0x89, 0xae, 0xa8, 0x65, 0x0c, 0x6c, 0xda, 0xcf, 0xb5, 0x02, 0x21, 0x78, 0x64, 0x3a, + 0xa1, 0xe3, 0x76, 0x99, 0xc1, 0x8a, 0x95, 0xaf, 0xc0, 0x6c, 0x22, 0xa6, 0xa2, 0x05, 0x98, 0x7d, + 0xd8, 0x6a, 0xb4, 0x1a, 0xbb, 0x8d, 0xea, 0x56, 0xe3, 0x7d, 0xaa, 0xf1, 0x19, 0x98, 0x6a, 0x36, + 0xda, 0xcd, 0xea, 0x6e, 0xed, 0xbe, 0xa6, 0x90, 0x99, 0xb1, 0x5f, 0xf5, 0xde, 0x5b, 0xbf, 0xfa, + 0x7c, 0x4d, 0xf9, 0xe4, 0xf3, 0x35, 0xe5, 0xb3, 0xcf, 0xd7, 0x94, 0xef, 0x7f, 0xb1, 0x76, 0xee, + 0x93, 0x2f, 0xd6, 0xce, 0x7d, 0xfa, 0xc5, 0xda, 0xb9, 0xf7, 0xaf, 0x77, 0x9d, 0x70, 0x7f, 0xb8, + 0x77, 0xc3, 0xf2, 0xfa, 0x37, 0x07, 0x8e, 0xdb, 0xb5, 0xcc, 0xc1, 0xcd, 0xd0, 0xb1, 0x6c, 0xeb, + 0xa6, 0xb4, 0x2c, 0xf7, 0x8a, 0xf4, 0xee, 0xe6, 0xb5, 0x7f, 0x04, 0x00, 0x00, 0xff, 0xff, 0xb8, + 0x77, 0xd4, 0x78, 0x07, 0x32, 0x00, 0x00, } func (m *TableSpan) Marshal() (dAtA []byte, err error) { @@ -5561,6 +5703,23 @@ func (m *NodeHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.NodeResourceUsageProtocolVersion != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeResourceUsageProtocolVersion)) + i-- + dAtA[i] = 0x48 + } + if m.NodeResourceUsage != nil { + { + size, err := m.NodeResourceUsage.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintHeartbeat(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x42 + } if m.WriteLeaseWitnessAck != nil { { size, err := m.WriteLeaseWitnessAck.MarshalToSizedBuffer(dAtA[:i]) @@ -5728,6 +5887,25 @@ func (m *NodeHeartbeatResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.NodeResourceUsageStatus != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeResourceUsageStatus)) + i-- + dAtA[i] = 0x38 + } + if len(m.NodeResourceUsages) > 0 { + for iNdEx := len(m.NodeResourceUsages) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.NodeResourceUsages[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintHeartbeat(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + } if m.WitnessChallenge != nil { { size, err := m.WitnessChallenge.MarshalToSizedBuffer(dAtA[:i]) @@ -6716,21 +6894,21 @@ func (m *InfluencedTables) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x18 } if len(m.TableIDs) > 0 { - dAtA43 := make([]byte, len(m.TableIDs)*10) - var j42 int + dAtA44 := make([]byte, len(m.TableIDs)*10) + var j43 int for _, num1 := range m.TableIDs { num := uint64(num1) for num >= 1<<7 { - dAtA43[j42] = uint8(uint64(num)&0x7f | 0x80) + dAtA44[j43] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j42++ + j43++ } - dAtA43[j42] = uint8(num) - j42++ + dAtA44[j43] = uint8(num) + j43++ } - i -= j42 - copy(dAtA[i:], dAtA43[:j42]) - i = encodeVarintHeartbeat(dAtA, i, uint64(j42)) + i -= j43 + copy(dAtA[i:], dAtA44[:j43]) + i = encodeVarintHeartbeat(dAtA, i, uint64(j43)) i-- dAtA[i] = 0x12 } @@ -7559,6 +7737,46 @@ func (m *DispatcherSetChecksumUpdateRequest) MarshalToSizedBuffer(dAtA []byte) ( return len(dAtA) - i, nil } +func (m *NodeResourceUsage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NodeResourceUsage) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NodeResourceUsage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.EventStoreWriteBytesPerSecond != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.EventStoreWriteBytesPerSecond)) + i-- + dAtA[i] = 0x18 + } + if len(m.NodeId) > 0 { + i -= len(m.NodeId) + copy(dAtA[i:], m.NodeId) + i = encodeVarintHeartbeat(dAtA, i, uint64(len(m.NodeId))) + i-- + dAtA[i] = 0x12 + } + if m.EventStoreWriteBytes != 0 { + i = encodeVarintHeartbeat(dAtA, i, uint64(m.EventStoreWriteBytes)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func encodeVarintHeartbeat(dAtA []byte, offset int, v uint64) int { offset -= sovHeartbeat(v) base := offset @@ -8036,6 +8254,13 @@ func (m *NodeHeartbeat) Size() (n int) { l = m.WriteLeaseWitnessAck.Size() n += 1 + l + sovHeartbeat(uint64(l)) } + if m.NodeResourceUsage != nil { + l = m.NodeResourceUsage.Size() + n += 1 + l + sovHeartbeat(uint64(l)) + } + if m.NodeResourceUsageProtocolVersion != 0 { + n += 1 + sovHeartbeat(uint64(m.NodeResourceUsageProtocolVersion)) + } return n } @@ -8111,6 +8336,15 @@ func (m *NodeHeartbeatResponse) Size() (n int) { l = m.WitnessChallenge.Size() n += 1 + l + sovHeartbeat(uint64(l)) } + if len(m.NodeResourceUsages) > 0 { + for _, e := range m.NodeResourceUsages { + l = e.Size() + n += 1 + l + sovHeartbeat(uint64(l)) + } + } + if m.NodeResourceUsageStatus != 0 { + n += 1 + sovHeartbeat(uint64(m.NodeResourceUsageStatus)) + } return n } @@ -8882,6 +9116,25 @@ func (m *DispatcherSetChecksumUpdateRequest) Size() (n int) { return n } +func (m *NodeResourceUsage) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.EventStoreWriteBytes != 0 { + n += 1 + sovHeartbeat(uint64(m.EventStoreWriteBytes)) + } + l = len(m.NodeId) + if l > 0 { + n += 1 + l + sovHeartbeat(uint64(l)) + } + if m.EventStoreWriteBytesPerSecond != 0 { + n += 1 + sovHeartbeat(uint64(m.EventStoreWriteBytesPerSecond)) + } + return n +} + func sovHeartbeat(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -11962,6 +12215,61 @@ func (m *NodeHeartbeat) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeResourceUsage", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthHeartbeat + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthHeartbeat + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.NodeResourceUsage == nil { + m.NodeResourceUsage = &NodeResourceUsage{} + } + if err := m.NodeResourceUsage.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeResourceUsageProtocolVersion", wireType) + } + m.NodeResourceUsageProtocolVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeResourceUsageProtocolVersion |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipHeartbeat(dAtA[iNdEx:]) @@ -12444,6 +12752,59 @@ func (m *NodeHeartbeatResponse) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeResourceUsages", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthHeartbeat + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthHeartbeat + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NodeResourceUsages = append(m.NodeResourceUsages, &NodeResourceUsage{}) + if err := m.NodeResourceUsages[len(m.NodeResourceUsages)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeResourceUsageStatus", wireType) + } + m.NodeResourceUsageStatus = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeResourceUsageStatus |= NodeResourceUsageStatus(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipHeartbeat(dAtA[iNdEx:]) @@ -17496,6 +17857,126 @@ func (m *DispatcherSetChecksumUpdateRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *NodeResourceUsage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeResourceUsage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeResourceUsage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EventStoreWriteBytes", wireType) + } + m.EventStoreWriteBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EventStoreWriteBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthHeartbeat + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthHeartbeat + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NodeId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EventStoreWriteBytesPerSecond", wireType) + } + m.EventStoreWriteBytesPerSecond = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EventStoreWriteBytesPerSecond |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipHeartbeat(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHeartbeat + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipHeartbeat(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/heartbeatpb/heartbeat.proto b/heartbeatpb/heartbeat.proto index 4f16a90539..4398b8855d 100644 --- a/heartbeatpb/heartbeat.proto +++ b/heartbeatpb/heartbeat.proto @@ -209,6 +209,8 @@ message NodeHeartbeat { uint64 write_lease_request_seq = 5; uint32 write_lease_protocol_version = 6; WriteLeaseWitnessAck write_lease_witness_ack = 7; + NodeResourceUsage node_resource_usage = 8; + uint32 node_resource_usage_protocol_version = 9; } message WriteLeaseWitnessChallenge { @@ -233,6 +235,16 @@ message NodeHeartbeatResponse { uint64 request_seq = 3; uint64 lease_duration_ms = 4; WriteLeaseWitnessChallenge witness_challenge = 5; + repeated NodeResourceUsage node_resource_usages = 6; + NodeResourceUsageStatus node_resource_usage_status = 7; +} + +// NodeResourceUsageStatus distinguishes rolling-upgrade compatibility from a +// telemetry interruption on nodes that have declared support for reporting. +enum NodeResourceUsageStatus { + UNSUPPORTED = 0; + INCOMPLETE = 1; + AVAILABLE = 2; } // SetNodeLivenessRequest asks a node to transition its local liveness. @@ -549,3 +561,15 @@ message DispatcherSetChecksumUpdateRequest { uint64 seq = 4; DispatcherSetChecksum checksum = 5; } + +// NodeResourceUsage carries node-wide EventStore resource usage. Node heartbeat +// requests report the cumulative counter; coordinator responses set node_id and +// the derived rate for every entry in the cluster snapshot. +message NodeResourceUsage { + // Set by node heartbeat requests. + uint64 event_store_write_bytes = 1; + // Set by coordinator heartbeat responses. + string node_id = 2; + // Set by coordinator heartbeat responses. + uint64 event_store_write_bytes_per_second = 3; +} diff --git a/heartbeatpb/node_resource_usage_protocol.go b/heartbeatpb/node_resource_usage_protocol.go new file mode 100644 index 0000000000..44df65ae39 --- /dev/null +++ b/heartbeatpb/node_resource_usage_protocol.go @@ -0,0 +1,19 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package heartbeatpb + +const ( + LegacyNodeResourceUsageProtocolVersion uint32 = 0 + CurrentNodeResourceUsageProtocolVersion uint32 = 2 +) diff --git a/logservice/eventstore/event_store.go b/logservice/eventstore/event_store.go index d85319b97c..8bfb20ed35 100644 --- a/logservice/eventstore/event_store.go +++ b/logservice/eventstore/event_store.go @@ -28,6 +28,7 @@ import ( "github.com/cockroachdb/pebble" "github.com/klauspost/compress/zstd" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller" @@ -98,6 +99,8 @@ type EventStore interface { GetIterator(dispatcherID common.DispatcherID, request ScanRequest) (EventIterator, error) GetLogCoordinatorNodeID() node.ID + + EventStoreWriteBytes() uint64 } type DMLEventState struct { @@ -267,6 +270,9 @@ type eventStore struct { // closed is used to indicate the event store is closed. closed atomic.Bool + // writeBytes is the authoritative process-wide cumulative write counter used + // by scheduling. The Prometheus counter remains an observability-only copy. + writeBytes atomic.Uint64 // compressionThreshold is the size in bytes above which a value will be compressed. compressionThreshold int @@ -1027,6 +1033,10 @@ func (e *eventStore) GetLogCoordinatorNodeID() node.ID { return e.getCoordinatorInfo() } +func (e *eventStore) EventStoreWriteBytes() uint64 { + return e.writeBytes.Load() +} + func (e *eventStore) detachFromSubStat(dispatcherID common.DispatcherID, subStat *subscriptionStat) { if subStat == nil { return @@ -1513,9 +1523,11 @@ func (e *eventStore) writeEvents( insertKVEntryCount.Add(float64(insertCount)) updateKVEntryCount.Add(float64(updateCount)) deleteKVEntryCount.Add(float64(deleteCount)) + writeBytes := uint64(batch.Len()) + failpoint.Inject("InjectEventStoreWriteBytes", func(val failpoint.Value) { + writeBytes = uint64(val.(int)) + }) metrics.EventStoreWriteBatchEventsCountHist.Observe(float64(kvCount)) - metrics.EventStoreWriteBatchSizeHist.Observe(float64(batch.Len())) - metrics.EventStoreWriteBytes.Add(float64(batch.Len())) if totalValueBytesAfter > 0 { metrics.EventStoreCompressionRatioHistogram.Observe(float64(totalValueBytesBefore) / float64(totalValueBytesAfter)) } @@ -1523,6 +1535,11 @@ func (e *eventStore) writeEvents( start := time.Now() err := batch.Commit(pebble.NoSync) metrics.EventStoreWriteDurationHistogram.Observe(time.Since(start).Seconds()) + if err == nil { + e.writeBytes.Add(writeBytes) + metrics.EventStoreWriteBatchSizeHist.Observe(float64(writeBytes)) + metrics.EventStoreWriteBytes.Add(float64(writeBytes)) + } return err } diff --git a/logservice/eventstore/event_store_test.go b/logservice/eventstore/event_store_test.go index 8e5380b3d2..91fdc89e1a 100644 --- a/logservice/eventstore/event_store_test.go +++ b/logservice/eventstore/event_store_test.go @@ -38,6 +38,7 @@ import ( "github.com/pingcap/ticdc/pkg/pdutil" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" ) @@ -1691,13 +1692,65 @@ func TestEventStoreKVEntryCount(t *testing.T) { encoder, err := zstd.NewWriter(nil) require.NoError(t, err) defer encoder.Close() + writeBytesBefore := store.EventStoreWriteBytes() require.NoError(t, store.writeEvents(store.dbs[0], events, encoder, nil, nil)) + require.Greater(t, store.EventStoreWriteBytes(), writeBytesBefore) for i, metric := range entryMetrics { require.Equal(t, before[i]+1, testutil.ToFloat64(metric)) } } +func TestEventStoreWriteBytesOnlyCountsCommittedBatches(t *testing.T) { + dir := t.TempDir() + _, storeInt := newEventStoreForTest(dir) + store := storeInt.(*eventStore) + defer store.Close(context.Background()) + + events := []eventWithCallback{{ + subID: 1, + tableID: 1, + kvs: []common.RawKVEntry{{ + OpType: common.OpTypePut, + StartTs: 1, + CRTs: 2, + Key: []byte("key"), + Value: []byte("value"), + }}, + callback: func() {}, + }} + encoder, err := zstd.NewWriter(nil) + require.NoError(t, err) + defer encoder.Close() + + readOnlyDir := t.TempDir() + db, err := pebble.Open(readOnlyDir, nil) + require.NoError(t, err) + require.NoError(t, db.Close()) + db, err = pebble.Open(readOnlyDir, &pebble.Options{ReadOnly: true}) + require.NoError(t, err) + defer db.Close() + + writeBytesBefore := store.EventStoreWriteBytes() + metricWriteBytesBefore := testutil.ToFloat64(metrics.EventStoreWriteBytes) + batchSizeBefore := &dto.Metric{} + require.NoError(t, metrics.EventStoreWriteBatchSizeHist.Write(batchSizeBefore)) + require.ErrorIs(t, store.writeEvents(db, events, encoder, nil, nil), pebble.ErrReadOnly) + require.Equal(t, writeBytesBefore, store.EventStoreWriteBytes()) + require.Equal(t, metricWriteBytesBefore, testutil.ToFloat64(metrics.EventStoreWriteBytes)) + batchSizeAfter := &dto.Metric{} + require.NoError(t, metrics.EventStoreWriteBatchSizeHist.Write(batchSizeAfter)) + require.Equal(t, batchSizeBefore.GetHistogram(), batchSizeAfter.GetHistogram()) + + require.NoError(t, store.writeEvents(store.dbs[0], events, encoder, nil, nil)) + writtenBytes := store.EventStoreWriteBytes() - writeBytesBefore + require.Positive(t, writtenBytes) + require.Equal(t, metricWriteBytesBefore+float64(writtenBytes), testutil.ToFloat64(metrics.EventStoreWriteBytes)) + require.NoError(t, metrics.EventStoreWriteBatchSizeHist.Write(batchSizeAfter)) + require.Equal(t, batchSizeBefore.GetHistogram().GetSampleCount()+1, batchSizeAfter.GetHistogram().GetSampleCount()) + require.Equal(t, batchSizeBefore.GetHistogram().GetSampleSum()+float64(writtenBytes), batchSizeAfter.GetHistogram().GetSampleSum()) +} + func TestEventStoreIterReadsLegacyCompressedValuesWithEncryptionManager(t *testing.T) { restoreCfg := setZstdCompressionForTest(t, true) defer restoreCfg() diff --git a/maintainer/barrier_event_test.go b/maintainer/barrier_event_test.go index 77e7465515..0c5c466e35 100644 --- a/maintainer/barrier_event_test.go +++ b/maintainer/barrier_event_test.go @@ -41,7 +41,7 @@ func TestScheduleEvent(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "test1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) event := NewBlockEvent(cfID, tableTriggerEventDispatcherID, spanController, operatorController, &heartbeatpb.State{ @@ -94,7 +94,7 @@ func TestResendAction(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 2}, 1) @@ -202,7 +202,7 @@ func TestFanoutPassResendWaitsForQuietStatus(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 2}, 1) @@ -256,7 +256,7 @@ func TestNormalPassResendDoesNotWaitForQuietStatus(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 2}, 1) @@ -298,7 +298,7 @@ func TestSendPassActionTypeDBIncludesWriterNode(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node2", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) @@ -341,7 +341,7 @@ func TestUpdateSchemaID(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) require.Equal(t, 1, spanController.GetAbsentSize()) diff --git a/maintainer/barrier_test.go b/maintainer/barrier_test.go index 2cbd0516f1..79090d38ef 100644 --- a/maintainer/barrier_test.go +++ b/maintainer/barrier_test.go @@ -47,7 +47,7 @@ func TestOneBlockEvent(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) startTs := uint64(10) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, startTs) @@ -179,7 +179,7 @@ func TestBarrierIgnoresBlockStatusFromNonOwner(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 10) stm := spanController.GetTasksByTableID(1)[0] @@ -222,7 +222,7 @@ func TestNormalBlock(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) var blockedDispatcherIDS []*heartbeatpb.DispatcherID for id := 1; id < 4; id++ { @@ -391,7 +391,7 @@ func TestNormalBlockWithTableTrigger(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) var blockedDispatcherIDS []*heartbeatpb.DispatcherID for id := 1; id < 3; id++ { @@ -623,7 +623,7 @@ func TestBarrierAppliesRecoveredRouteEventBeforeActionResend(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 10) stm := spanController.GetTasksByTableID(1)[0] @@ -681,7 +681,7 @@ func TestBarrierCommitsForwardedRouteEventBeforeLaterRouteEvent(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 10) tableSpan := spanController.GetTasksByTableID(1)[0] @@ -894,7 +894,7 @@ func newBarrierRoutePrecheckTestFixture( ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 10) stm := spanController.GetTasksByTableID(1)[0] @@ -1017,7 +1017,7 @@ func TestSchemaBlock(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) @@ -1193,7 +1193,7 @@ func TestSyncPointBlock(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 1) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 2}, 1) @@ -1366,7 +1366,7 @@ func TestNonBlocked(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) barrier := NewBarrier(spanController, operatorController, false, nil, common.DefaultMode, nil) @@ -1421,7 +1421,7 @@ func TestUpdateCheckpointTs(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) barrier := NewBarrier(spanController, operatorController, false, nil, common.DefaultMode, nil) msgs := barrier.HandleStatus("node1", &heartbeatpb.BlockStatusRequest{ @@ -1476,7 +1476,7 @@ func TestHandleBlockBootstrapResponse(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) var dispatcherIDs []*heartbeatpb.DispatcherID @@ -1637,7 +1637,7 @@ func TestSyncPointBlockPerf(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) barrier := NewBarrier(spanController, operatorController, true, nil, common.DefaultMode, nil) for id := 1; id < 1000; id++ { @@ -1717,7 +1717,7 @@ func TestBarrierEventWithDispatcherReallocation(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) tableID := int64(1) @@ -1924,7 +1924,7 @@ func TestBarrierEventWithDispatcherScheduling(t *testing.T) { ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) // Setup dispatcher A @@ -2061,7 +2061,7 @@ func TestDeferAllDBBlockEventFromDDLDispatcherWhilePendingSchedule(t *testing.T) ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) oldTableID := int64(1) @@ -2173,7 +2173,7 @@ func TestBarrierReturnsTemporaryIgnoreForUnreplicatingWaitingStatus(t *testing.T ComponentStatus: heartbeatpb.ComponentState_Working, CheckpointTs: 1, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1000, common.DefaultMode) spanController.AddNewTable(commonEvent.Table{SchemaID: 1, TableID: 1}, 10) diff --git a/maintainer/maintainer.go b/maintainer/maintainer.go index 786c167427..a510c4d920 100644 --- a/maintainer/maintainer.go +++ b/maintainer/maintainer.go @@ -180,6 +180,7 @@ func NewMaintainer(cfID common.ChangeFeedID, checkpointTs uint64, newChangefeed bool, keyspaceID uint32, + nodeResourceUsage *replica.NodeResourceUsageTracker, ) *Maintainer { mc := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter) nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) @@ -208,7 +209,7 @@ func NewMaintainer(cfID common.ChangeFeedID, checkpointUpdateCh: make(chan struct{}, 1), startCheckpointTs: checkpointTs, controller: NewController(cfID, checkpointTs, taskScheduler, - info.Config, ddlSpan, redoDDLSpan, conf.AddTableBatchSize, time.Duration(conf.CheckBalanceInterval), refresher, keyspaceMeta, enableRedo, conf.BalanceMoveBatchSize, info.Epoch), + info.Config, ddlSpan, redoDDLSpan, conf.AddTableBatchSize, time.Duration(conf.CheckBalanceInterval), refresher, keyspaceMeta, enableRedo, conf.BalanceMoveBatchSize, info.Epoch, nodeResourceUsage), mc: mc, removed: atomic.NewBool(false), nodeManager: nodeManager, @@ -292,6 +293,7 @@ func NewMaintainerForRemove(cfID common.ChangeFeedID, taskScheduler threadpool.ThreadPool, keyspaceID uint32, maintainerEpoch uint64, + nodeResourceUsage *replica.NodeResourceUsageTracker, ) *Maintainer { unused := &config.ChangeFeedInfo{ ChangefeedID: cfID, @@ -299,7 +301,7 @@ func NewMaintainerForRemove(cfID common.ChangeFeedID, Config: config.GetDefaultReplicaConfig(), Epoch: maintainerEpoch, } - m := NewMaintainer(cfID, conf, unused, selfNode, taskScheduler, 1, false, keyspaceID) + m := NewMaintainer(cfID, conf, unused, selfNode, taskScheduler, 1, false, keyspaceID, nodeResourceUsage) m.cascadeRemoving.Store(true) return m } diff --git a/maintainer/maintainer_controller.go b/maintainer/maintainer_controller.go index 8371444968..8a6fe65456 100644 --- a/maintainer/maintainer_controller.go +++ b/maintainer/maintainer_controller.go @@ -98,6 +98,7 @@ func NewController(changefeedID common.ChangeFeedID, enableRedo bool, balanceMoveBatchSize int, maintainerEpoch uint64, + nodeResourceUsage *replica.NodeResourceUsageTracker, ) *Controller { mc := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter) @@ -117,14 +118,18 @@ func NewController(changefeedID common.ChangeFeedID, if replicaConfig != nil { schedulerCfg = replicaConfig.Scheduler } - spanController := span.NewController(changefeedID, ddlSpan, splitter, schedulerCfg, refresher, keyspaceMeta.ID, common.DefaultMode) + spanController := span.NewController( + changefeedID, ddlSpan, splitter, schedulerCfg, refresher, + keyspaceMeta.ID, common.DefaultMode, nodeResourceUsage) var ( redoSpanController *span.Controller redoOC *operator.Controller ) if enableRedo { - redoSpanController = span.NewController(changefeedID, redoDDLSpan, splitter, schedulerCfg, refresher, keyspaceMeta.ID, common.RedoMode) + redoSpanController = span.NewController( + changefeedID, redoDDLSpan, splitter, schedulerCfg, refresher, + keyspaceMeta.ID, common.RedoMode, nodeResourceUsage) redoOC = operator.NewOperatorController(changefeedID, redoSpanController, batchSize, common.RedoMode) } // Create operator controller using spanController diff --git a/maintainer/maintainer_controller_test.go b/maintainer/maintainer_controller_test.go index 28316c2ea2..04d945b17d 100644 --- a/maintainer/maintainer_controller_test.go +++ b/maintainer/maintainer_controller_test.go @@ -76,7 +76,7 @@ func TestSchedule(t *testing.T) { CheckpointTs: 1, }, "node1", false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) - controller := NewController(cfID, 1, nil, replicaConfig, ddlSpan, nil, 9, time.Minute, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + controller := NewController(cfID, 1, nil, replicaConfig, ddlSpan, nil, 9, time.Minute, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) for i := 0; i < 10; i++ { controller.spanController.AddNewTable(commonEvent.Table{ SchemaID: 1, @@ -131,6 +131,7 @@ func TestNewControllerInitializesMaintainerEpoch(t *testing.T) { true, testBalanceMoveBatchSize, maintainerEpoch, + replica.NewNodeResourceUsageTracker(), ) require.Equal(t, maintainerEpoch, controller.currentMaintainerEpoch()) @@ -156,6 +157,8 @@ func TestBalanceGroupsNewNodeAdd_SplitsTableMoreThanNodeNum(t *testing.T) { CheckpointTs: 1, }, "node1", false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) + resourceUsageTracker := replica.NewNodeResourceUsageTracker() + resourceUsageTracker.ReplaceEventStoreWriteBytesPerSecond(nil, heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED) s := NewController(cfID, 1, nil, &config.ReplicaConfig{ Scheduler: &config.ChangefeedSchedulerConfig{ EnableTableAcrossNodes: util.AddressOf(true), @@ -164,7 +167,7 @@ func TestBalanceGroupsNewNodeAdd_SplitsTableMoreThanNodeNum(t *testing.T) { MinTrafficPercentage: util.AddressOf(0.8), MaxTrafficPercentage: util.AddressOf(1.2), }, - }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, resourceUsageTracker) nodeID := node.ID("node1") for i := range 100 { @@ -288,6 +291,8 @@ func TestBalanceGroupsNewNodeAdd_SplitsTableLessThanNodeNum(t *testing.T) { CheckpointTs: 1, }, "node1", false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) + resourceUsageTracker := replica.NewNodeResourceUsageTracker() + resourceUsageTracker.ReplaceEventStoreWriteBytesPerSecond(nil, heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED) s := NewController(cfID, 1, nil, &config.ReplicaConfig{ Scheduler: &config.ChangefeedSchedulerConfig{ EnableTableAcrossNodes: util.AddressOf(true), @@ -296,7 +301,7 @@ func TestBalanceGroupsNewNodeAdd_SplitsTableLessThanNodeNum(t *testing.T) { MinTrafficPercentage: util.AddressOf(0.8), MaxTrafficPercentage: util.AddressOf(1.2), }, - }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, resourceUsageTracker) regionCache := appcontext.GetService[*testutil.MockCache](appcontext.RegionCache) @@ -418,7 +423,7 @@ func TestSplitBalanceGroupsWithNodeRemove(t *testing.T) { MinTrafficPercentage: util.AddressOf(0.8), MaxTrafficPercentage: util.AddressOf(1.2), }, - }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) nodeIDList := []node.ID{"node1", "node2", "node3"} for i := 0; i < 100; i++ { @@ -510,6 +515,8 @@ func TestSplitTableBalanceWhenTrafficUnbalanced(t *testing.T) { }, "node1", false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) + resourceUsageTracker := replica.NewNodeResourceUsageTracker() + resourceUsageTracker.ReplaceEventStoreWriteBytesPerSecond(nil, heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED) controller := NewController(cfID, 1, nil, &config.ReplicaConfig{ Scheduler: &config.ChangefeedSchedulerConfig{ EnableTableAcrossNodes: util.AddressOf(true), @@ -520,7 +527,7 @@ func TestSplitTableBalanceWhenTrafficUnbalanced(t *testing.T) { MinTrafficPercentage: util.AddressOf(0.8), MaxTrafficPercentage: util.AddressOf(1.2), }, - }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, resourceUsageTracker) nodeIDList := []node.ID{"node1", "node2", "node3"} // make a group @@ -1121,7 +1128,7 @@ func TestBalance(t *testing.T) { CheckpointTs: 1, }, "node1", false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) - s := NewController(cfID, 1, nil, replicaConfig, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + s := NewController(cfID, 1, nil, replicaConfig, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) for i := 0; i < 100; i++ { sz := common.TableIDToComparableSpan(common.DefaultKeyspaceID, int64(i)) span := &heartbeatpb.TableSpan{TableID: sz.TableID, StartKey: sz.StartKey, EndKey: sz.EndKey} @@ -1227,7 +1234,7 @@ func TestDefaultSpanIntoSplit(t *testing.T) { MinTrafficPercentage: util.AddressOf(0.8), MaxTrafficPercentage: util.AddressOf(1.2), }, - }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) totalSpan := common.TableIDToComparableSpan(common.DefaultKeyspaceID, 1) span := &heartbeatpb.TableSpan{TableID: int64(1), StartKey: totalSpan.StartKey, EndKey: totalSpan.EndKey} dispatcherID := common.NewDispatcherID() @@ -1369,7 +1376,7 @@ func TestStoppedWhenMoving(t *testing.T) { CheckpointTs: 1, }, "node1", false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) - s := NewController(cfID, 1, nil, replicaConfig, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + s := NewController(cfID, 1, nil, replicaConfig, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) for i := 0; i < 2; i++ { sz := common.TableIDToComparableSpan(common.DefaultKeyspaceID, int64(i)) span := &heartbeatpb.TableSpan{TableID: sz.TableID, StartKey: sz.StartKey, EndKey: sz.EndKey} @@ -1422,7 +1429,7 @@ func TestFinishBootstrap(t *testing.T) { }, "node1", false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) s := NewController(cfID, 1, &mockThreadPool{}, - config.GetDefaultReplicaConfig(), ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + config.GetDefaultReplicaConfig(), ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) totalSpan := common.TableIDToComparableSpan(common.DefaultKeyspaceID, 1) span := &heartbeatpb.TableSpan{TableID: int64(1), StartKey: totalSpan.StartKey, EndKey: totalSpan.EndKey} schemaStore := eventservice.NewMockSchemaStore() @@ -1495,7 +1502,7 @@ func TestFinishBootstrapReturnsErrorWhenCheckpointMissing(t *testing.T) { }, "node1", false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) controller := NewController(cfID, 1, &mockThreadPool{}, - config.GetDefaultReplicaConfig(), ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + config.GetDefaultReplicaConfig(), ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) postBootstrapRequest, err := controller.FinishBootstrap(map[node.ID]*heartbeatpb.MaintainerBootstrapResponse{ "node1": { @@ -1559,7 +1566,7 @@ func TestFinishBootstrapSkipsStaleCreateOperatorForDroppedTable(t *testing.T) { }, "node1", false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) s := NewController(cfID, 1, &mockThreadPool{}, - config.GetDefaultReplicaConfig(), ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + config.GetDefaultReplicaConfig(), ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) // The schema-store snapshot is empty at bootstrap startTs, which models a table // that has already been dropped before failover recovery starts. @@ -2048,7 +2055,7 @@ func newMergeBootstrapTestEnv(t *testing.T) *mergeBootstrapTestEnv { MaxTrafficPercentage: util.AddressOf(1.2), } refresher := replica.NewRegionCountRefresher(cfID, time.Minute) - controller := NewController(cfID, 1, &mockThreadPool{}, cfg, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + controller := NewController(cfID, 1, &mockThreadPool{}, cfg, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) schemaStore := eventservice.NewMockSchemaStore() schemaStore.SetTables([]commonEvent.Table{ @@ -2325,7 +2332,7 @@ func TestSplitTableWhenBootstrapFinished(t *testing.T) { MaxTrafficPercentage: util.AddressOf(1.2), } refresher := replica.NewRegionCountRefresher(cfID, time.Minute) - s := NewController(cfID, 1, nil, defaultConfig, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + s := NewController(cfID, 1, nil, defaultConfig, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) s.taskPool = &mockThreadPool{} schemaStore := eventservice.NewMockSchemaStore() schemaStore.SetTables( @@ -2513,7 +2520,7 @@ func TestLargeTableInitialization(t *testing.T) { MinTrafficPercentage: util.AddressOf(0.8), MaxTrafficPercentage: util.AddressOf(1.2), }, - }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0) + }, ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 0, replica.NewNodeResourceUsageTracker()) // Create a large table with 10000 regions totalSpan := common.TableIDToComparableSpan(common.DefaultKeyspaceID, int64(1)) diff --git a/maintainer/maintainer_manager.go b/maintainer/maintainer_manager.go index acc7c523ff..791f3c09f0 100644 --- a/maintainer/maintainer_manager.go +++ b/maintainer/maintainer_manager.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/maintainer/replica" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/config" @@ -33,6 +34,12 @@ const ( defaultManagerHeartbeatInterval = 200 * time.Millisecond ) +// NodeResourceUsageProvider exposes exact process-wide counters used by the +// scheduler. Implementations must not depend on Prometheus metric collection. +type NodeResourceUsageProvider interface { + EventStoreWriteBytes() uint64 +} + // Manager is the manager of all changefeed maintainer in a ticdc server, each ticdc server will // start a Manager when the ticdc server is startup. It responsible for: // 1. Handle bootstrap command from coordinator and report all changefeed maintainer status. @@ -52,6 +59,9 @@ type Manager struct { heartbeatCh chan struct{} // node holds node-scoped liveness and drain state that applies to the whole capture. node *managerNodeState + // nodeResourceUsage is the cluster snapshot shared by every local maintainer. + nodeResourceUsage *replica.NodeResourceUsageTracker + resourceUsageProvider NodeResourceUsageProvider // maintainers holds changefeed-scoped state and lifecycle operations. maintainers *managerMaintainerSet writeGate *writelease.Gate @@ -66,6 +76,7 @@ func NewMaintainerManager( nodeInfo *node.Info, conf *config.SchedulerConfig, nodeLiveness *liveness.Liveness, + resourceUsageProvider NodeResourceUsageProvider, ) *Manager { mc := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter) heartbeatCh := make(chan struct{}, 1) @@ -73,14 +84,17 @@ func NewMaintainerManager( if !ok { writeGate = writelease.NewGate() } + nodeResourceUsage := replica.NewNodeResourceUsageTracker() m := &Manager{ - mc: mc, - nodeInfo: nodeInfo, - msgCh: make(chan *messaging.TargetMessage, 1024), - heartbeatCh: heartbeatCh, - node: newManagerNodeState(nodeLiveness), - maintainers: newManagerMaintainerSet(conf, nodeInfo, heartbeatCh), - writeGate: writeGate, + mc: mc, + nodeInfo: nodeInfo, + msgCh: make(chan *messaging.TargetMessage, 1024), + heartbeatCh: heartbeatCh, + node: newManagerNodeState(nodeLiveness), + nodeResourceUsage: nodeResourceUsage, + resourceUsageProvider: resourceUsageProvider, + maintainers: newManagerMaintainerSet(conf, nodeInfo, heartbeatCh, nodeResourceUsage), + writeGate: writeGate, } mc.RegisterHandler(messaging.MaintainerManagerTopic, m.recvMessages) diff --git a/maintainer/maintainer_manager_maintainers.go b/maintainer/maintainer_manager_maintainers.go index 0f1916e280..5860bc99e5 100644 --- a/maintainer/maintainer_manager_maintainers.go +++ b/maintainer/maintainer_manager_maintainers.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/maintainer/replica" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" @@ -52,6 +53,8 @@ type managerMaintainerSet struct { taskScheduler threadpool.ThreadPool // heartbeatCh coalesces prompt reports from low-latency maintainers. heartbeatCh chan<- struct{} + // nodeResourceUsage is shared by all local changefeed maintainers. + nodeResourceUsage *replica.NodeResourceUsageTracker // registryMu serializes registry mutations that create, replace, or fully // close maintainers because maintainer metrics share changefeed labels across @@ -66,12 +69,14 @@ func newManagerMaintainerSet( conf *config.SchedulerConfig, nodeInfo *node.Info, heartbeatCh chan<- struct{}, + nodeResourceUsage *replica.NodeResourceUsageTracker, ) *managerMaintainerSet { return &managerMaintainerSet{ - conf: conf, - nodeInfo: nodeInfo, - taskScheduler: threadpool.NewThreadPoolDefault(), - heartbeatCh: heartbeatCh, + conf: conf, + nodeInfo: nodeInfo, + taskScheduler: threadpool.NewThreadPoolDefault(), + heartbeatCh: heartbeatCh, + nodeResourceUsage: nodeResourceUsage, } } @@ -218,7 +223,7 @@ func (p *managerMaintainerSet) handleAddMaintainer( // Create the maintainer only after epoch admission so normal duplicate // add retries do not start short-lived goroutines or metrics. newMaintainer := func() *Maintainer { - maintainer := NewMaintainer(changefeedID, p.conf, info, p.nodeInfo, p.taskScheduler, req.CheckpointTs, req.IsNewChangefeed, req.KeyspaceId) + maintainer := NewMaintainer(changefeedID, p.conf, info, p.nodeInfo, p.taskScheduler, req.CheckpointTs, req.IsNewChangefeed, req.KeyspaceId, p.nodeResourceUsage) maintainer.managerHeartbeatCh = p.heartbeatCh return maintainer } @@ -354,6 +359,7 @@ func (p *managerMaintainerSet) handleRemoveMaintainer(msg *messaging.TargetMessa p.taskScheduler, req.KeyspaceId, req.MaintainerEpoch, + p.nodeResourceUsage, ) p.registry.Store(changefeedID, maintainer) } diff --git a/maintainer/maintainer_manager_node.go b/maintainer/maintainer_manager_node.go index 2b53f96a70..5cda6ec262 100644 --- a/maintainer/maintainer_manager_node.go +++ b/maintainer/maintainer_manager_node.go @@ -108,11 +108,15 @@ func (m *Manager) sendNodeHeartbeat(force bool) { NodeEpoch: m.node.nodeEpoch, // Report the manager-level dispatcher drain target so coordinator can // confirm both activation and clearing even when no maintainers exist. - DispatcherDrainTargetNodeId: drainTarget.String(), - DispatcherDrainTargetEpoch: drainEpoch, - WriteLeaseRequestSeq: requestSeq, - WriteLeaseProtocolVersion: heartbeatpb.CurrentWriteLeaseProtocolVersion, - WriteLeaseWitnessAck: m.node.pendingWitnessAck, + DispatcherDrainTargetNodeId: drainTarget.String(), + DispatcherDrainTargetEpoch: drainEpoch, + WriteLeaseRequestSeq: requestSeq, + WriteLeaseProtocolVersion: heartbeatpb.CurrentWriteLeaseProtocolVersion, + WriteLeaseWitnessAck: m.node.pendingWitnessAck, + NodeResourceUsageProtocolVersion: heartbeatpb.CurrentNodeResourceUsageProtocolVersion, + NodeResourceUsage: &heartbeatpb.NodeResourceUsage{ + EventStoreWriteBytes: m.resourceUsageProvider.EventStoreWriteBytes(), + }, } target := m.newCoordinatorTopicMessage(hb) if err := m.mc.SendCommand(target); err != nil { @@ -190,6 +194,8 @@ func (m *Manager) onNodeHeartbeatResponse(msg *messaging.TargetMessage) { } m.writeGate.SetP2PRequired(true) } + m.nodeResourceUsage.ReplaceEventStoreWriteBytesPerSecond( + response.NodeResourceUsages, response.NodeResourceUsageStatus) metrics.CaptureLeaseResponseCounter.WithLabelValues("accepted").Inc() m.node.lastAppliedLeaseSeq = requestSeq for seq := range m.node.writeLeaseRequestSentAt { diff --git a/maintainer/maintainer_manager_test.go b/maintainer/maintainer_manager_test.go index 9c3f1abe54..426b501053 100644 --- a/maintainer/maintainer_manager_test.go +++ b/maintainer/maintainer_manager_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/maintainer/replica" "github.com/pingcap/ticdc/maintainer/testutil" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" @@ -103,7 +104,8 @@ func newManagerMaintainerSetForAddTest(t *testing.T) *managerMaintainerSet { testutil.SetUpTestServices(t) selfNode := node.NewInfo("", "") - maintainers := newManagerMaintainerSet(config.NewDefaultSchedulerConfig(), selfNode, nil) + maintainers := newManagerMaintainerSet( + config.NewDefaultSchedulerConfig(), selfNode, nil, replica.NewNodeResourceUsageTracker()) t.Cleanup(maintainers.closeAll) return maintainers } @@ -325,7 +327,7 @@ func TestMaintainerSchedulesNodeChanges(t *testing.T) { schedulerConf.AddTableBatchSize = 1000 schedulerConf.CheckBalanceInterval = 0 var nodeLiveness liveness.Liveness - manager := NewMaintainerManager(selfNode, schedulerConf, &nodeLiveness) + manager := NewMaintainerManager(selfNode, schedulerConf, &nodeLiveness, fixedNodeResourceUsageProvider(0)) msg := messaging.NewSingleTargetMessage(selfNode.ID, messaging.MaintainerManagerTopic, &heartbeatpb.CoordinatorBootstrapRequest{Version: 1}) @@ -553,7 +555,7 @@ func TestMaintainerBootstrapWithTablesReported(t *testing.T) { return nil }) var nodeLiveness liveness.Liveness - manager := NewMaintainerManager(selfNode, config.GetGlobalServerConfig().Debug.Scheduler, &nodeLiveness) + manager := NewMaintainerManager(selfNode, config.GetGlobalServerConfig().Debug.Scheduler, &nodeLiveness, fixedNodeResourceUsageProvider(0)) msg := messaging.NewSingleTargetMessage(selfNode.ID, messaging.MaintainerManagerTopic, &heartbeatpb.CoordinatorBootstrapRequest{Version: 1}) @@ -699,7 +701,7 @@ func TestStopNotExistsMaintainer(t *testing.T) { schedulerConf := config.NewDefaultSchedulerConfig() schedulerConf.AddTableBatchSize = 1000 var nodeLiveness liveness.Liveness - manager := NewMaintainerManager(selfNode, schedulerConf, &nodeLiveness) + manager := NewMaintainerManager(selfNode, schedulerConf, &nodeLiveness, fixedNodeResourceUsageProvider(0)) msg := messaging.NewSingleTargetMessage(selfNode.ID, messaging.MaintainerManagerTopic, &heartbeatpb.CoordinatorBootstrapRequest{Version: 1}) diff --git a/maintainer/maintainer_test.go b/maintainer/maintainer_test.go index 9578c6680e..8d08c7bcc5 100644 --- a/maintainer/maintainer_test.go +++ b/maintainer/maintainer_test.go @@ -385,7 +385,7 @@ func TestMaintainerSchedule(t *testing.T) { }, &config.ChangeFeedInfo{ Config: config.GetDefaultReplicaConfig(), - }, n, taskScheduler, 10, true, common.DefaultKeyspaceID) + }, n, taskScheduler, 10, true, common.DefaultKeyspaceID, replica.NewNodeResourceUsageTracker()) defer maintainer.Close() mc.RegisterHandler(messaging.MaintainerManagerTopic, @@ -443,7 +443,7 @@ func TestMaintainer_GetMaintainerStatusUsesCommittedCheckpoint(t *testing.T) { CheckpointTs: 10, Mode: common.DefaultMode, }, "node1", false) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) spanController.AdvanceMaintainerCommittedCheckpointTs(20) m := &Maintainer{ @@ -488,7 +488,7 @@ func TestMaintainerHeartbeatDuringRemovingSkipsFailoverRecovery(t *testing.T) { }, captureID, false) refresher := replica.NewRegionCountRefresher(cfID, time.Minute) controller := NewController(cfID, 10, &mockThreadPool{}, - config.GetDefaultReplicaConfig(), ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 1) + config.GetDefaultReplicaConfig(), ddlSpan, nil, 1000, 0, refresher, common.DefaultKeyspace, false, testBalanceMoveBatchSize, 1, replica.NewNodeResourceUsageTracker()) totalSpan := common.TableIDToComparableSpan(common.DefaultKeyspaceID, 1) dispatcherID := common.NewDispatcherID() @@ -814,7 +814,7 @@ func newMaintainerForCheckpointCalculationTest(t testing.TB) (*Maintainer, node. cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) selfNode := node.NewInfo("127.0.0.1:8300", "") _, ddlSpan := newDDLSpan(common.DefaultKeyspaceID, cfID, 1, selfNode, common.DefaultMode) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1, common.DefaultMode) controller := &Controller{ @@ -862,9 +862,9 @@ func newMaintainerForRedoCheckpointCalculationTest(t testing.TB) (*Maintainer, n selfNode := node.NewInfo("127.0.0.1:8300", "") _, ddlSpan := newDDLSpan(common.DefaultKeyspaceID, cfID, 1, selfNode, common.DefaultMode) _, redoDDLSpan := newDDLSpan(common.DefaultKeyspaceID, cfID, 1, selfNode, common.RedoMode) - spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) operatorController := operator.NewOperatorController(cfID, spanController, 1, common.DefaultMode) - redoSpanController := span.NewController(cfID, redoDDLSpan, nil, nil, nil, common.DefaultKeyspaceID, common.RedoMode) + redoSpanController := span.NewController(cfID, redoDDLSpan, nil, nil, nil, common.DefaultKeyspaceID, common.RedoMode, replica.NewNodeResourceUsageTracker()) redoOperatorController := operator.NewOperatorController(cfID, redoSpanController, 1, common.RedoMode) controller := &Controller{ diff --git a/maintainer/node_liveness_test.go b/maintainer/node_liveness_test.go index 89f1c92a9b..241fe3ff3c 100644 --- a/maintainer/node_liveness_test.go +++ b/maintainer/node_liveness_test.go @@ -28,12 +28,18 @@ import ( "github.com/stretchr/testify/require" ) +type fixedNodeResourceUsageProvider uint64 + +func (p fixedNodeResourceUsageProvider) EventStoreWriteBytes() uint64 { + return uint64(p) +} + func TestSetNodeLivenessRejectEpochMismatch(t *testing.T) { mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 @@ -59,7 +65,7 @@ func TestSetNodeLivenessApplyTransition(t *testing.T) { appcontext.SetService(appcontext.MessageCenter, mc) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 @@ -87,7 +93,7 @@ func TestSetDispatcherDrainTargetApplyAndClear(t *testing.T) { appcontext.SetService(appcontext.MessageCenter, mc) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) m.coordinatorID = node.ID("coordinator") msg := messaging.NewSingleTargetMessage( @@ -124,7 +130,7 @@ func TestSetDispatcherDrainTargetRejectStaleUpdate(t *testing.T) { appcontext.SetService(appcontext.MessageCenter, mc) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) m.coordinatorID = node.ID("coordinator") apply := func(target string, epoch uint64) { @@ -163,7 +169,7 @@ func TestSetDispatcherDrainTargetSendsNodeHeartbeatAck(t *testing.T) { appcontext.SetService(appcontext.MessageCenter, mc) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 1 @@ -204,7 +210,7 @@ func TestCoordinatorBootstrapResponseIncludesDispatcherDrainTarget(t *testing.T) appcontext.SetService(appcontext.MessageCenter, mc) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) require.True(t, m.node.tryUpdateDispatcherDrainTarget(node.ID("n2"), 7)) req := messaging.NewSingleTargetMessage( @@ -230,7 +236,7 @@ func TestCoordinatorBootstrapNegotiatesP2PWriteLease(t *testing.T) { appcontext.SetService(appcontext.CaptureWriteGate, gate) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration)) require.True(t, gate.IsWritable()) @@ -270,7 +276,7 @@ func TestNodeHeartbeatResponseRenewsP2PWriteLease(t *testing.T) { appcontext.SetService(appcontext.CaptureWriteGate, gate) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 10 gate.SetP2PRequired(true) @@ -309,6 +315,78 @@ func TestNodeHeartbeatResponseRenewsP2PWriteLease(t *testing.T) { require.False(t, gate.IsWritable()) } +func TestNodeHeartbeatReportsAndReceivesResourceUsage(t *testing.T) { + mc := messaging.NewMockMessageCenter() + appcontext.SetService(appcontext.MessageCenter, mc) + gate := writelease.NewGate() + appcontext.SetService(appcontext.CaptureWriteGate, gate) + + var nodeLiveness liveness.Liveness + m := NewMaintainerManager( + &node.Info{ID: node.ID("n1")}, + &config.SchedulerConfig{}, + &nodeLiveness, + fixedNodeResourceUsageProvider(123), + ) + m.coordinatorID = node.ID("coordinator") + m.coordinatorVersion = 10 + + m.sendNodeHeartbeat(true) + heartbeatMessage := <-mc.GetMessageChannel() + heartbeat := heartbeatMessage.Message[0].(*heartbeatpb.NodeHeartbeat) + require.Equal(t, uint64(123), heartbeat.GetNodeResourceUsage().GetEventStoreWriteBytes()) + require.Empty(t, heartbeat.GetNodeResourceUsage().GetNodeId()) + require.Equal(t, heartbeatpb.CurrentNodeResourceUsageProtocolVersion, + heartbeat.GetNodeResourceUsageProtocolVersion()) + + responseMessage := messaging.NewSingleTargetMessage( + m.nodeInfo.ID, + messaging.MaintainerManagerTopic, + &heartbeatpb.NodeHeartbeatResponse{ + CoordinatorVersion: 10, + TargetNodeEpoch: m.node.nodeEpoch, + RequestSeq: heartbeat.WriteLeaseRequestSeq, + LeaseDurationMs: 0, + NodeResourceUsages: []*heartbeatpb.NodeResourceUsage{ + {NodeId: "n1", EventStoreWriteBytesPerSecond: 10}, + {NodeId: "n2", EventStoreWriteBytesPerSecond: 20}, + }, + NodeResourceUsageStatus: heartbeatpb.NodeResourceUsageStatus_AVAILABLE, + }, + ) + responseMessage.From = m.coordinatorID + m.onNodeHeartbeatResponse(responseMessage) + + rate, status, _ := m.nodeResourceUsage.EventStoreWriteBytesPerSecond([]node.ID{"n1", "n2"}) + require.Equal(t, heartbeatpb.NodeResourceUsageStatus_AVAILABLE, status) + require.Equal(t, map[node.ID]uint64{"n1": 10, "n2": 20}, rate) + + m.sendNodeHeartbeat(true) + heartbeatMessage = <-mc.GetMessageChannel() + heartbeat = heartbeatMessage.Message[0].(*heartbeatpb.NodeHeartbeat) + responseMessage = messaging.NewSingleTargetMessage( + m.nodeInfo.ID, + messaging.MaintainerManagerTopic, + &heartbeatpb.NodeHeartbeatResponse{ + CoordinatorVersion: 10, + TargetNodeEpoch: m.node.nodeEpoch, + RequestSeq: heartbeat.WriteLeaseRequestSeq, + LeaseDurationMs: 0, + NodeResourceUsages: []*heartbeatpb.NodeResourceUsage{ + {NodeId: "n1", EventStoreWriteBytesPerSecond: 30}, + {NodeId: "n2", EventStoreWriteBytesPerSecond: 40}, + }, + NodeResourceUsageStatus: heartbeatpb.NodeResourceUsageStatus_AVAILABLE, + }, + ) + responseMessage.From = m.coordinatorID + m.onNodeHeartbeatResponse(responseMessage) + + rate, status, _ = m.nodeResourceUsage.EventStoreWriteBytesPerSecond([]node.ID{"n1", "n2"}) + require.Equal(t, heartbeatpb.NodeResourceUsageStatus_AVAILABLE, status) + require.Equal(t, map[node.ID]uint64{"n1": 30, "n2": 40}, rate) +} + func TestNodeHeartbeatResponseUpdatesClusterP2PMode(t *testing.T) { mc := messaging.NewMockMessageCenter() appcontext.SetService(appcontext.MessageCenter, mc) @@ -316,7 +394,7 @@ func TestNodeHeartbeatResponseUpdatesClusterP2PMode(t *testing.T) { appcontext.SetService(appcontext.CaptureWriteGate, gate) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 10 require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration)) @@ -364,7 +442,7 @@ func TestNodeHeartbeatResponseEchoesWitnessChallenge(t *testing.T) { appcontext.SetService(appcontext.CaptureWriteGate, writelease.NewGate()) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) m.coordinatorID = node.ID("coordinator") m.coordinatorVersion = 10 @@ -403,7 +481,7 @@ func TestAddMaintainerIgnoreInvalidConfig(t *testing.T) { appcontext.SetService(appcontext.MessageCenter, mc) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) changefeedID := common.NewChangeFeedIDWithName("cf-invalid-config", common.DefaultKeyspaceName) status := m.onAddMaintainerRequest(&heartbeatpb.AddMaintainerRequest{ @@ -422,7 +500,7 @@ func TestAddMaintainerIgnoreInvalidCheckpointTs(t *testing.T) { appcontext.SetService(appcontext.MessageCenter, mc) var nodeLiveness liveness.Liveness - m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness) + m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness, fixedNodeResourceUsageProvider(0)) changefeedID := common.NewChangeFeedIDWithName("cf-invalid-checkpoint", common.DefaultKeyspaceName) info := &config.ChangeFeedInfo{ diff --git a/maintainer/operator/operator_move_test.go b/maintainer/operator/operator_move_test.go index 6961a2695f..33fdf32cfa 100644 --- a/maintainer/operator/operator_move_test.go +++ b/maintainer/operator/operator_move_test.go @@ -65,7 +65,7 @@ func setupTestEnvironment(t *testing.T) (*span.Controller, common.ChangeFeedID, ) refresher := replica.NewRegionCountRefresher(changefeedID, time.Minute) - spanController := span.NewController(changefeedID, ddlSpan, nil, nil, refresher, common.DefaultKeyspaceID, common.DefaultMode) + spanController := span.NewController(changefeedID, ddlSpan, nil, nil, refresher, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) replicaSet := replica.NewWorkingSpanReplication( changefeedID, diff --git a/maintainer/replica/checker.go b/maintainer/replica/checker.go index 692e186d8b..6099fa4d66 100644 --- a/maintainer/replica/checker.go +++ b/maintainer/replica/checker.go @@ -34,17 +34,21 @@ func GetNewGroupChecker( cfID common.ChangeFeedID, schedulerCfg *config.ChangefeedSchedulerConfig, refresher *RegionCountRefresher, + nodeResourceUsage *NodeResourceUsageTracker, ) func(replica.GroupID) replica.GroupChecker[common.DispatcherID, *SpanReplication] { if schedulerCfg == nil || !util.GetOrZero(schedulerCfg.EnableTableAcrossNodes) { return replica.NewEmptyChecker[common.DispatcherID, *SpanReplication] } + eventStoreBalanceLimiter := &eventStoreBalanceLimiter{} return func(groupID replica.GroupID) replica.GroupChecker[common.DispatcherID, *SpanReplication] { groupType := replica.GetGroupType(groupID) switch groupType { case replica.GroupDefault: return NewDefaultSpanSplitChecker(cfID, schedulerCfg, refresher) case replica.GroupTable: - return NewSplitSpanChecker(cfID, groupID, schedulerCfg, refresher) + return NewSplitSpanChecker( + cfID, groupID, schedulerCfg, refresher, + nodeResourceUsage, eventStoreBalanceLimiter) } log.Panic("unknown group type", zap.String("changefeed", cfID.Name()), zap.Int8("groupType", int8(groupType))) return nil diff --git a/maintainer/replica/node_resource_usage.go b/maintainer/replica/node_resource_usage.go new file mode 100644 index 0000000000..265a731b2c --- /dev/null +++ b/maintainer/replica/node_resource_usage.go @@ -0,0 +1,107 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package replica + +import ( + "sync" + "time" + + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/node" +) + +// Resource usage is normally reported by the node heartbeat every 500ms. +// Allow several missed reports before treating current-version telemetry as +// incomplete and suppressing traffic-driven moves. +const nodeResourceUsageStaleThreshold = 5 * time.Second + +// NodeResourceUsageTracker stores one immutable rate snapshot shared by all +// local changefeed group checkers. +type NodeResourceUsageTracker struct { + mu sync.RWMutex + eventStoreWriteBytesPerSecond map[node.ID]uint64 + status heartbeatpb.NodeResourceUsageStatus + generation uint64 + updatedAt time.Time + now func() time.Time +} + +func NewNodeResourceUsageTracker() *NodeResourceUsageTracker { + return &NodeResourceUsageTracker{ + status: heartbeatpb.NodeResourceUsageStatus_INCOMPLETE, + now: time.Now, + } +} + +// ReplaceEventStoreWriteBytesPerSecond atomically replaces the cluster-wide +// rate snapshot computed by the coordinator. +func (t *NodeResourceUsageTracker) ReplaceEventStoreWriteBytesPerSecond( + usages []*heartbeatpb.NodeResourceUsage, + status heartbeatpb.NodeResourceUsageStatus, +) { + t.mu.Lock() + defer t.mu.Unlock() + + now := t.now() + t.generation++ + if status != heartbeatpb.NodeResourceUsageStatus_AVAILABLE { + if status != heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED { + status = heartbeatpb.NodeResourceUsageStatus_INCOMPLETE + } + t.eventStoreWriteBytesPerSecond = nil + t.status = status + t.updatedAt = now + return + } + + rates := make(map[node.ID]uint64, len(usages)) + for _, usage := range usages { + if usage == nil || usage.NodeId == "" { + t.eventStoreWriteBytesPerSecond = nil + t.status = heartbeatpb.NodeResourceUsageStatus_INCOMPLETE + t.updatedAt = now + return + } + nodeID := node.ID(usage.NodeId) + rates[nodeID] = usage.EventStoreWriteBytesPerSecond + } + t.updatedAt = now + t.eventStoreWriteBytesPerSecond = rates + t.status = heartbeatpb.NodeResourceUsageStatus_AVAILABLE +} + +// EventStoreWriteBytesPerSecond returns a shared immutable rate snapshot, its +// availability, and a generation that advances when the snapshot is replaced. +// Unsupported means callers may use the legacy policy during a rolling upgrade. +// Incomplete means resource-aware moves must be suppressed. +func (t *NodeResourceUsageTracker) EventStoreWriteBytesPerSecond( + nodeIDs []node.ID, +) (map[node.ID]uint64, heartbeatpb.NodeResourceUsageStatus, uint64) { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.status == heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED { + return nil, t.status, t.generation + } + if t.status != heartbeatpb.NodeResourceUsageStatus_AVAILABLE || + t.now().Sub(t.updatedAt) > nodeResourceUsageStaleThreshold { + return nil, heartbeatpb.NodeResourceUsageStatus_INCOMPLETE, t.generation + } + for _, nodeID := range nodeIDs { + if _, ok := t.eventStoreWriteBytesPerSecond[nodeID]; !ok { + return nil, heartbeatpb.NodeResourceUsageStatus_INCOMPLETE, t.generation + } + } + return t.eventStoreWriteBytesPerSecond, heartbeatpb.NodeResourceUsageStatus_AVAILABLE, t.generation +} diff --git a/maintainer/replica/node_resource_usage_test.go b/maintainer/replica/node_resource_usage_test.go new file mode 100644 index 0000000000..554adfd7bf --- /dev/null +++ b/maintainer/replica/node_resource_usage_test.go @@ -0,0 +1,76 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package replica + +import ( + "testing" + "time" + + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/node" + "github.com/stretchr/testify/require" +) + +func TestNodeResourceUsageTrackerSharesRateSnapshot(t *testing.T) { + now := time.Unix(100, 0) + tracker := NewNodeResourceUsageTracker() + tracker.now = func() time.Time { return now } + nodeIDs := []node.ID{"node1", "node2"} + + tracker.ReplaceEventStoreWriteBytesPerSecond([]*heartbeatpb.NodeResourceUsage{ + {NodeId: "node1", EventStoreWriteBytesPerSecond: 10}, + {NodeId: "node2", EventStoreWriteBytesPerSecond: 20}, + }, heartbeatpb.NodeResourceUsageStatus_AVAILABLE) + rate, status, generation := tracker.EventStoreWriteBytesPerSecond(nodeIDs) + require.Equal(t, heartbeatpb.NodeResourceUsageStatus_AVAILABLE, status) + require.Equal(t, map[node.ID]uint64{"node1": 10, "node2": 20}, rate) + require.Equal(t, uint64(1), generation) + + rateAgain, status, generationAgain := tracker.EventStoreWriteBytesPerSecond(nodeIDs) + require.Equal(t, heartbeatpb.NodeResourceUsageStatus_AVAILABLE, status) + require.Equal(t, rate, rateAgain) + require.Equal(t, generation, generationAgain) + require.Zero(t, testing.AllocsPerRun(100, func() { + tracker.EventStoreWriteBytesPerSecond(nodeIDs) + })) + + now = now.Add(nodeResourceUsageStaleThreshold + time.Nanosecond) + _, status, generationAgain = tracker.EventStoreWriteBytesPerSecond(nodeIDs) + require.Equal(t, heartbeatpb.NodeResourceUsageStatus_INCOMPLETE, status) + require.Equal(t, generation, generationAgain) + + tracker.ReplaceEventStoreWriteBytesPerSecond([]*heartbeatpb.NodeResourceUsage{ + {NodeId: "node1", EventStoreWriteBytesPerSecond: 10}, + {NodeId: "node2", EventStoreWriteBytesPerSecond: 20}, + }, heartbeatpb.NodeResourceUsageStatus_AVAILABLE) + _, status, generationAgain = tracker.EventStoreWriteBytesPerSecond(nodeIDs) + require.Equal(t, heartbeatpb.NodeResourceUsageStatus_AVAILABLE, status) + require.Equal(t, generation+1, generationAgain) +} + +func TestNodeResourceUsageTrackerDistinguishesUnsupportedAndIncomplete(t *testing.T) { + tracker := NewNodeResourceUsageTracker() + nodeIDs := []node.ID{"node1", "node2"} + + _, status, _ := tracker.EventStoreWriteBytesPerSecond(nodeIDs) + require.Equal(t, heartbeatpb.NodeResourceUsageStatus_INCOMPLETE, status) + + tracker.ReplaceEventStoreWriteBytesPerSecond(nil, heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED) + _, status, _ = tracker.EventStoreWriteBytesPerSecond(nodeIDs) + require.Equal(t, heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED, status) + + tracker.ReplaceEventStoreWriteBytesPerSecond(nil, heartbeatpb.NodeResourceUsageStatus_INCOMPLETE) + _, status, _ = tracker.EventStoreWriteBytesPerSecond(nodeIDs) + require.Equal(t, heartbeatpb.NodeResourceUsageStatus_INCOMPLETE, status) +} diff --git a/maintainer/replica/split_span_checker.go b/maintainer/replica/split_span_checker.go index 366ca3144b..7afbea8350 100644 --- a/maintainer/replica/split_span_checker.go +++ b/maintainer/replica/split_span_checker.go @@ -40,7 +40,15 @@ import ( "go.uber.org/zap" ) -const latestTrafficIndex = 0 +const ( + latestTrafficIndex = 0 + trafficBalanceSkipResourceUsageIncomplete = "resource_usage_incomplete" + trafficBalanceSkipNoEventStoreHeadroom = "no_event_store_headroom" + trafficBalanceSkipNoImprovingSpan = "no_improving_span" + // Ignore small relative skews that do not indicate meaningful storage pressure. + minEventStoreWriteBytesPerSecond = 10 * 1024 * 1024 + eventStoreFollowUpBalanceScoreThreshold = 3 +) var ( minTrafficBalanceThreshold = float64(1024 * 1024) // 1MB @@ -74,6 +82,12 @@ type BalanceCondition struct { statusUpdated bool } +type eventStoreBalanceLimiter struct { + generation uint64 + lastMoveSnapshotGeneration uint64 + evacuatingSourceNodeID node.ID +} + func (b *BalanceCondition) reset() { b.minTrafficNodeID = "" b.maxTrafficNodeID = "" @@ -153,7 +167,11 @@ type SplitSpanChecker struct { minTrafficPercentage float64 maxTrafficPercentage float64 - balanceCondition BalanceCondition + balanceCondition BalanceCondition + eventStoreBalanceCondition BalanceCondition + eventStoreBalanceGeneration uint64 + eventStoreSnapshotGeneration uint64 + eventStoreBalanceLimiter *eventStoreBalanceLimiter mergeThreshold int mergeCheckCount int @@ -161,6 +179,8 @@ type SplitSpanChecker struct { nodeManager *watcher.NodeManager pdClock pdutil.Clock + nodeResourceUsage *NodeResourceUsageTracker + refresher *RegionCountRefresher splitSpanCheckDuration prometheus.Observer } @@ -181,24 +201,28 @@ func NewSplitSpanChecker( groupID replica.GroupID, schedulerCfg *config.ChangefeedSchedulerConfig, refresher *RegionCountRefresher, + nodeResourceUsage *NodeResourceUsageTracker, + eventStoreBalanceLimiter *eventStoreBalanceLimiter, ) *SplitSpanChecker { if schedulerCfg == nil { log.Panic("scheduler config is nil, please check the config", zap.String("changefeed", changefeedID.Name())) } return &SplitSpanChecker{ - changefeedID: changefeedID, - groupID: groupID, - allTasks: make(map[common.DispatcherID]*splitSpanStatus), - writeThreshold: util.GetOrZero(schedulerCfg.WriteKeyThreshold), - regionThreshold: util.GetOrZero(schedulerCfg.RegionThreshold), - balanceScoreThreshold: util.GetOrZero(schedulerCfg.BalanceScoreThreshold), - minTrafficPercentage: util.GetOrZero(schedulerCfg.MinTrafficPercentage), - maxTrafficPercentage: util.GetOrZero(schedulerCfg.MaxTrafficPercentage), - mergeThreshold: mergeThreshold, - mergeCheckCount: 0, - nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), - pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), - splitSpanCheckDuration: metrics.SplitSpanCheckDuration.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), replica.GetGroupName(groupID)), + changefeedID: changefeedID, + groupID: groupID, + allTasks: make(map[common.DispatcherID]*splitSpanStatus), + writeThreshold: util.GetOrZero(schedulerCfg.WriteKeyThreshold), + regionThreshold: util.GetOrZero(schedulerCfg.RegionThreshold), + balanceScoreThreshold: util.GetOrZero(schedulerCfg.BalanceScoreThreshold), + minTrafficPercentage: util.GetOrZero(schedulerCfg.MinTrafficPercentage), + maxTrafficPercentage: util.GetOrZero(schedulerCfg.MaxTrafficPercentage), + eventStoreBalanceLimiter: eventStoreBalanceLimiter, + mergeThreshold: mergeThreshold, + mergeCheckCount: 0, + nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), + pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), + nodeResourceUsage: nodeResourceUsage, + splitSpanCheckDuration: metrics.SplitSpanCheckDuration.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), replica.GetGroupName(groupID)), refresher: refresher, } @@ -262,6 +286,7 @@ func (s *SplitSpanChecker) UpdateStatus(replica *SpanReplication) { } s.balanceCondition.statusUpdated = true + s.eventStoreBalanceCondition.statusUpdated = true log.Debug("split span checker update status", zap.Any("changefeedID", s.changefeedID), @@ -325,6 +350,8 @@ func (s *SplitSpanChecker) Check(batch int) replica.GroupCheckResult { } aliveNodeIDs := s.nodeManager.GetAliveNodeIDs() + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus, eventStoreSnapshotGeneration := + s.nodeResourceUsage.EventStoreWriteBytesPerSecond(aliveNodeIDs) lastThreeTrafficPerNode := make(map[node.ID][]float64) lastThreeTrafficSum := make([]float64, 3) @@ -344,22 +371,36 @@ func (s *SplitSpanChecker) Check(batch int) replica.GroupCheckResult { return results } - // step2. check whether the whole dispatchers should be merged together. + // step2. move a dispatcher away from a node with excessive EventStore traffic. + var eventStoreBalanceNeeded bool + results, eventStoreBalanceNeeded = s.checkBalanceEventStore( + aliveNodeIDs, taskMap, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus, + eventStoreSnapshotGeneration) + if len(results) > 0 || eventStoreBalanceNeeded { + return results + } + + // step3. check whether the whole dispatchers should be merged together. // only when all spans' total region count and traffic are less then threshold/2, we can merge them together // consider we only support to merge the spans in the same node, we first do move, then merge - results = s.checkMergeWhole(totalRegionCount, lastThreeTrafficSum, lastThreeTrafficPerNode) + results = s.checkMergeWhole( + totalRegionCount, lastThreeTrafficSum, lastThreeTrafficPerNode, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus) if len(results) > 0 { return results } - // step3. check the traffic of each node. If the traffic is not balanced, + // step4. check the traffic of each node. If the traffic is not balanced, // we try to move some spans from the node with max traffic to the node with min traffic - results, minTrafficNodeID, maxTrafficNodeID := s.checkBalanceTraffic(aliveNodeIDs, lastThreeTrafficSum, lastThreeTrafficPerNode, taskMap) + results, minTrafficNodeID, maxTrafficNodeID := s.checkBalanceTraffic( + aliveNodeIDs, lastThreeTrafficSum, lastThreeTrafficPerNode, taskMap, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus) if len(results) > 0 { return results } - // step4. check whether we need to do merge some spans. + // step5. check whether we need to do merge some spans. // we can only merge spans when the lag is low. minCheckpointTs := uint64(math.MaxUint64) for _, status := range s.allTasks { @@ -424,7 +465,10 @@ func (s *SplitSpanChecker) Check(batch int) replica.GroupCheckResult { } // step5. try to check whether we need move dispatchers to make merge possible - return s.chooseMoveSpans(minTrafficNodeID, maxTrafficNodeID, sortedSpanByStartKey, lastThreeTrafficPerNode, taskMap) + return s.chooseMoveSpans( + minTrafficNodeID, maxTrafficNodeID, sortedSpanByStartKey, + lastThreeTrafficPerNode, taskMap, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus) } // chooseMoveSpans finds multiple optimal span moves using a multi-priority search strategy: @@ -432,7 +476,15 @@ func (s *SplitSpanChecker) Check(batch int) replica.GroupCheckResult { // 2. Priority 2: Merge moves with compensation to maintain balance // 3. Priority 3: Pure traffic balance moves // Returns multiple move plans sorted by priority and effectiveness -func (s *SplitSpanChecker) chooseMoveSpans(minTrafficNodeID node.ID, maxTrafficNodeID node.ID, sortedSpanByStartKey []*splitSpanStatus, lastThreeTrafficPerNode map[node.ID][]float64, taskMap map[node.ID][]*splitSpanStatus) []SplitSpanCheckResult { +func (s *SplitSpanChecker) chooseMoveSpans( + minTrafficNodeID node.ID, + maxTrafficNodeID node.ID, + sortedSpanByStartKey []*splitSpanStatus, + lastThreeTrafficPerNode map[node.ID][]float64, + taskMap map[node.ID][]*splitSpanStatus, + eventStoreWriteBytesPerSecond map[node.ID]uint64, + nodeResourceUsageStatus heartbeatpb.NodeResourceUsageStatus, +) []SplitSpanCheckResult { log.Debug("chooseMoveSpans try to choose move spans", zap.Any("changefeedID", s.changefeedID), zap.Any("groupID", s.groupID), @@ -444,6 +496,12 @@ func (s *SplitSpanChecker) chooseMoveSpans(minTrafficNodeID node.ID, maxTrafficN // If no any span in minTrafficNodeID, we random select one span from maxTrafficNodeID for it. if len(taskMap[minTrafficNodeID]) == 0 && len(taskMap[maxTrafficNodeID]) > 0 { + if !eventStoreDestinationEligible( + maxTrafficNodeID, minTrafficNodeID, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus, + ) { + return results + } randomSpan := taskMap[maxTrafficNodeID][rand.Intn(len(taskMap[maxTrafficNodeID]))] results = append(results, SplitSpanCheckResult{ OpType: OpMove, @@ -459,7 +517,11 @@ func (s *SplitSpanChecker) chooseMoveSpans(minTrafficNodeID node.ID, maxTrafficN adjacencyMap := s.buildAdjacencyMap(sortedSpanByStartKey) // Try to find optimal span moves using multi-priority strategy - if result := s.findOptimalSpanMoves(minTrafficNodeID, maxTrafficNodeID, sortedSpanByStartKey, adjacencyMap, lastThreeTrafficPerNode); len(result) > 0 { + if result := s.findOptimalSpanMoves( + minTrafficNodeID, maxTrafficNodeID, sortedSpanByStartKey, + adjacencyMap, lastThreeTrafficPerNode, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus, + ); len(result) > 0 { results = append(results, result...) log.Info("chooseMoveSpans found multiple move plans", zap.Any("changefeedID", s.changefeedID), @@ -472,6 +534,23 @@ func (s *SplitSpanChecker) chooseMoveSpans(minTrafficNodeID node.ID, maxTrafficN return results } +func eventStoreDestinationEligible( + sourceNodeID node.ID, + targetNodeID node.ID, + eventStoreWriteBytesPerSecond map[node.ID]uint64, + nodeResourceUsageStatus heartbeatpb.NodeResourceUsageStatus, +) bool { + if nodeResourceUsageStatus == heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED { + return true + } + if nodeResourceUsageStatus != heartbeatpb.NodeResourceUsageStatus_AVAILABLE { + return false + } + sourceWriteBytes, sourceOK := eventStoreWriteBytesPerSecond[sourceNodeID] + targetWriteBytes, targetOK := eventStoreWriteBytesPerSecond[targetNodeID] + return sourceOK && targetOK && targetWriteBytes < sourceWriteBytes +} + // buildAdjacencyMap builds a map from span ID to its adjacent spans for O(1) lookup func (s *SplitSpanChecker) buildAdjacencyMap(sortedSpanByStartKey []*splitSpanStatus) map[common.DispatcherID][]*splitSpanStatus { adjacencyMap := make(map[common.DispatcherID][]*splitSpanStatus) @@ -509,6 +588,8 @@ func (s *SplitSpanChecker) findOptimalSpanMoves( sortedSpanByStartKey []*splitSpanStatus, adjacencyMap map[common.DispatcherID][]*splitSpanStatus, lastThreeTrafficPerNode map[node.ID][]float64, + eventStoreWriteBytesPerSecond map[node.ID]uint64, + nodeResourceUsageStatus heartbeatpb.NodeResourceUsageStatus, ) []SplitSpanCheckResult { moveSpanTarget := make(map[common.DispatcherID]node.ID) // moveSpanTarget is used to store the target node for the spans ready to move @@ -552,6 +633,12 @@ func (s *SplitSpanChecker) findOptimalSpanMoves( if span.GetNodeID() == targetNodeID || selectedSpans[span.ID] { continue } + if !eventStoreDestinationEligible( + span.GetNodeID(), targetNodeID, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus, + ) { + continue + } // Check if moving this span to targetNodeID would enable merge if mergedSpans, ok := s.canMergeAfterMove(span, targetNodeID, adjacencyMap, moveSpanTarget); ok { @@ -583,7 +670,8 @@ func (s *SplitSpanChecker) findOptimalSpanMoves( // Single move violates balance, try to find compensation move if compensationMove, compSpan := s.findCompensationMove(span, mergedSpans, targetNodeID, span.GetNodeID(), sortedSpanByStartKey, afterMoveTrafficPerNode, - currentMinTraffic, currentMaxTraffic, selectedSpans, moveSpanTarget); compensationMove != nil { + currentMinTraffic, currentMaxTraffic, selectedSpans, moveSpanTarget, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus); compensationMove != nil { results = append(results, SplitSpanCheckResult{ OpType: OpMove, MoveSpans: []*SpanReplication{ @@ -630,6 +718,8 @@ func (s *SplitSpanChecker) findCompensationMove( currentMaxTraffic float64, selectedSpans map[common.DispatcherID]bool, moveSpanTarget map[common.DispatcherID]node.ID, + eventStoreWriteBytesPerSecond map[node.ID]uint64, + nodeResourceUsageStatus heartbeatpb.NodeResourceUsageStatus, ) (*SplitSpanCheckResult, *splitSpanStatus) { movedTraffic := movedSpan.lastThreeTraffic[latestTrafficIndex] @@ -654,6 +744,12 @@ func (s *SplitSpanChecker) findCompensationMove( // Look for spans in targetNode that can be moved to sourceNode for _, span := range targetNodeSpans { + if !eventStoreDestinationEligible( + span.GetNodeID(), sourceNode, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus, + ) { + continue + } // Try to find a span that can balance traffic // Note: compensation move doesn't need to enable merge, its main purpose is traffic balancing compensationTraffic := span.lastThreeTraffic[latestTrafficIndex] @@ -874,7 +970,13 @@ func (s *SplitSpanChecker) chooseMergedSpans(batchSize int) ([]SplitSpanCheckRes // check whether the whole dispatchers should be merged together. // only when all spans' total region count and traffic are less then threshold/2, we can merge them together // consider we only support to merge the spans in the same node, we first do move, then merge -func (s *SplitSpanChecker) checkMergeWhole(totalRegionCount int, lastThreeTrafficSum []float64, lastThreeTrafficPerNode map[node.ID][]float64) []SplitSpanCheckResult { +func (s *SplitSpanChecker) checkMergeWhole( + totalRegionCount int, + lastThreeTrafficSum []float64, + lastThreeTrafficPerNode map[node.ID][]float64, + eventStoreWriteBytesPerSecond map[node.ID]uint64, + nodeResourceUsageStatus heartbeatpb.NodeResourceUsageStatus, +) []SplitSpanCheckResult { log.Debug("checkMergeWhole try to merge whole spans", zap.Any("changefeedID", s.changefeedID), zap.Any("groupID", s.groupID), @@ -921,6 +1023,31 @@ func (s *SplitSpanChecker) checkMergeWhole(totalRegionCount int, lastThreeTraffi results = append(results, ret) return results } + if nodeResourceUsageStatus == heartbeatpb.NodeResourceUsageStatus_INCOMPLETE { + return results + } + if nodeResourceUsageStatus == heartbeatpb.NodeResourceUsageStatus_AVAILABLE { + targetNode = "" + for nodeID, traffic := range lastThreeTrafficPerNode { + if traffic[latestTrafficIndex] == 0 { + continue + } + if targetNode == "" || + eventStoreWriteBytesPerSecond[nodeID] < eventStoreWriteBytesPerSecond[targetNode] || + (eventStoreWriteBytesPerSecond[nodeID] == eventStoreWriteBytesPerSecond[targetNode] && nodeID < targetNode) { + targetNode = nodeID + } + } + for nodeID, traffic := range lastThreeTrafficPerNode { + if traffic[latestTrafficIndex] > 0 && nodeID != targetNode && + !eventStoreDestinationEligible( + nodeID, targetNode, + eventStoreWriteBytesPerSecond, nodeResourceUsageStatus, + ) { + return results + } + } + } // move all spans to the targetNode ret := SplitSpanCheckResult{ @@ -1031,6 +1158,136 @@ func (s *SplitSpanChecker) chooseSplitSpans( return results, totalRegionCount } +// checkBalanceEventStore moves one dispatcher away from a node whose +// EventStore traffic is persistently above the cluster average. +func (s *SplitSpanChecker) checkBalanceEventStore( + aliveNodeIDs []node.ID, + taskMap map[node.ID][]*splitSpanStatus, + eventStoreWriteBytesPerSecond map[node.ID]uint64, + nodeResourceUsageStatus heartbeatpb.NodeResourceUsageStatus, + eventStoreSnapshotGeneration uint64, +) ([]SplitSpanCheckResult, bool) { + results := make([]SplitSpanCheckResult, 0, 1) + if s.eventStoreBalanceGeneration != s.eventStoreBalanceLimiter.generation { + s.eventStoreBalanceCondition.reset() + s.eventStoreBalanceGeneration = s.eventStoreBalanceLimiter.generation + } + if nodeResourceUsageStatus != heartbeatpb.NodeResourceUsageStatus_AVAILABLE || + len(aliveNodeIDs) < 2 { + s.eventStoreBalanceCondition.reset() + s.eventStoreBalanceLimiter.evacuatingSourceNodeID = "" + return results, false + } + var totalWriteBytes float64 + var sourceNodeID node.ID + var sourceWriteBytes uint64 + for _, nodeID := range aliveNodeIDs { + writeBytes := eventStoreWriteBytesPerSecond[nodeID] + totalWriteBytes += float64(writeBytes) + if sourceNodeID == "" || writeBytes > sourceWriteBytes || + (writeBytes == sourceWriteBytes && nodeID < sourceNodeID) { + sourceNodeID = nodeID + sourceWriteBytes = writeBytes + } + } + avgWriteBytes := totalWriteBytes / float64(len(aliveNodeIDs)) + if sourceWriteBytes < minEventStoreWriteBytesPerSecond || + avgWriteBytes == 0 || float64(sourceWriteBytes) <= avgWriteBytes*s.maxTrafficPercentage { + s.eventStoreBalanceCondition.reset() + s.eventStoreBalanceLimiter.evacuatingSourceNodeID = "" + return results, false + } + if s.eventStoreBalanceLimiter.evacuatingSourceNodeID != "" && + s.eventStoreBalanceLimiter.evacuatingSourceNodeID != sourceNodeID { + s.eventStoreBalanceLimiter.evacuatingSourceNodeID = "" + } + if len(taskMap[sourceNodeID]) == 0 { + s.eventStoreBalanceCondition.reset() + return results, false + } + // EventStore traffic is node-wide and shared by all split-table groups. Do + // not emit multiple moves from one snapshot or count repeated checker runs + // as additional overload observations. + if eventStoreSnapshotGeneration == s.eventStoreBalanceLimiter.lastMoveSnapshotGeneration { + s.eventStoreBalanceCondition.reset() + s.balanceCondition.reset() + return results, true + } + if eventStoreSnapshotGeneration == s.eventStoreSnapshotGeneration { + return results, true + } + s.eventStoreSnapshotGeneration = eventStoreSnapshotGeneration + + var targetNodeID node.ID + var moveSpan *splitSpanStatus + bestAfterMoveDifference := math.MaxFloat64 + hasTarget := false + for _, nodeID := range aliveNodeIDs { + writeBytes := eventStoreWriteBytesPerSecond[nodeID] + if nodeID == sourceNodeID || float64(writeBytes) >= avgWriteBytes { + continue + } + hasTarget = true + beforeMoveDifference := float64(sourceWriteBytes - writeBytes) + for _, candidate := range taskMap[sourceNodeID] { + spanTraffic := candidate.lastThreeTraffic[latestTrafficIndex] + afterMoveDifference := math.Abs(beforeMoveDifference - 2*spanTraffic) + if spanTraffic <= 0 || afterMoveDifference >= beforeMoveDifference { + continue + } + if moveSpan == nil || afterMoveDifference < bestAfterMoveDifference || + (afterMoveDifference == bestAfterMoveDifference && + (nodeID < targetNodeID || + (nodeID == targetNodeID && candidate.ID.String() < moveSpan.ID.String()))) { + targetNodeID = nodeID + moveSpan = candidate + bestAfterMoveDifference = afterMoveDifference + } + } + } + if !hasTarget { + metrics.TrafficBalanceSkipCounter.WithLabelValues( + trafficBalanceSkipNoEventStoreHeadroom).Inc() + s.eventStoreBalanceCondition.reset() + return results, true + } + if moveSpan == nil { + metrics.TrafficBalanceSkipCounter.WithLabelValues( + trafficBalanceSkipNoImprovingSpan).Inc() + s.eventStoreBalanceCondition.reset() + return results, true + } + + s.eventStoreBalanceCondition.updateScore(targetNodeID, sourceNodeID, false, true) + balanceScoreThreshold := s.balanceScoreThreshold + if s.eventStoreBalanceLimiter.evacuatingSourceNodeID == sourceNodeID { + balanceScoreThreshold = min(balanceScoreThreshold, eventStoreFollowUpBalanceScoreThreshold) + } + if s.eventStoreBalanceCondition.balanceScore < balanceScoreThreshold { + return results, true + } + log.Info("move dispatcher away from busy event store", + zap.String("changefeed", s.changefeedID.String()), + zap.Int64("group", s.groupID), + zap.String("dispatcherID", moveSpan.ID.String()), + zap.Stringer("sourceNodeID", sourceNodeID), + zap.Stringer("targetNodeID", targetNodeID), + zap.Uint64("sourceWriteBytesPerSecond", sourceWriteBytes), + zap.Uint64("targetWriteBytesPerSecond", eventStoreWriteBytesPerSecond[targetNodeID])) + results = append(results, SplitSpanCheckResult{ + OpType: OpMove, + MoveSpans: []*SpanReplication{moveSpan.SpanReplication}, + TargetNode: targetNodeID, + }) + s.eventStoreBalanceLimiter.generation++ + s.eventStoreBalanceLimiter.lastMoveSnapshotGeneration = eventStoreSnapshotGeneration + s.eventStoreBalanceLimiter.evacuatingSourceNodeID = sourceNodeID + s.eventStoreBalanceGeneration = s.eventStoreBalanceLimiter.generation + s.eventStoreBalanceCondition.reset() + s.balanceCondition.reset() + return results, true +} + // checkBalanceTraffic checks whether the traffic is balanced for each node. // If the traffic is not balanced, we try to move some spans from the node with max traffic to the node with min traffic // If not existing spans can be moved, we try to split a span from the node with max traffic. @@ -1039,6 +1296,8 @@ func (s *SplitSpanChecker) checkBalanceTraffic( lastThreeTrafficSum []float64, lastThreeTrafficPerNode map[node.ID][]float64, taskMap map[node.ID][]*splitSpanStatus, + eventStoreWriteBytesPerSecond map[node.ID]uint64, + nodeResourceUsageStatus heartbeatpb.NodeResourceUsageStatus, ) (results []SplitSpanCheckResult, minTrafficNodeID node.ID, maxTrafficNodeID node.ID) { log.Debug("checkBalanceTraffic try to balance traffic", zap.Any("changefeedID", s.changefeedID), @@ -1152,10 +1411,42 @@ func (s *SplitSpanChecker) checkBalanceTraffic( } } + targetNodeID := minTrafficNodeID + if nodeResourceUsageStatus == heartbeatpb.NodeResourceUsageStatus_INCOMPLETE { + metrics.TrafficBalanceSkipCounter.WithLabelValues( + trafficBalanceSkipResourceUsageIncomplete).Inc() + s.balanceCondition.reset() + return + } + if nodeResourceUsageStatus == heartbeatpb.NodeResourceUsageStatus_AVAILABLE { + // Restrict candidates to nodes below this group's average traffic, then + // require less node-wide EventStore work than the source. Ranking alone + // is insufficient when every otherwise eligible destination is busier. + var targetWriteBytes uint64 + targetNodeID = "" + for _, nodeID := range aliveNodeIDs { + if nodeID == maxTrafficNodeID || + lastThreeTrafficPerNode[nodeID][latestTrafficIndex] >= avgLastThreeTraffic[latestTrafficIndex] || + eventStoreWriteBytesPerSecond[nodeID] >= eventStoreWriteBytesPerSecond[maxTrafficNodeID] { + continue + } + if targetNodeID == "" || eventStoreWriteBytesPerSecond[nodeID] < targetWriteBytes { + targetNodeID = nodeID + targetWriteBytes = eventStoreWriteBytesPerSecond[nodeID] + } + } + if targetNodeID == "" { + metrics.TrafficBalanceSkipCounter.WithLabelValues( + trafficBalanceSkipNoEventStoreHeadroom).Inc() + s.balanceCondition.reset() + return + } + } + // calculate the diff traffic between the avg traffic and the min/max traffic // we try to move spans, whose total traffic is close to diffTraffic, // from the max node to min node - diffInMinNode := avgLastThreeTraffic[latestTrafficIndex] - lastThreeTrafficPerNode[minTrafficNodeID][latestTrafficIndex] + diffInMinNode := avgLastThreeTraffic[latestTrafficIndex] - lastThreeTrafficPerNode[targetNodeID][latestTrafficIndex] diffInMaxNode := lastThreeTrafficPerNode[maxTrafficNodeID][latestTrafficIndex] - avgLastThreeTraffic[latestTrafficIndex] diffTraffic := math.Min(diffInMinNode, diffInMaxNode) @@ -1196,12 +1487,12 @@ func (s *SplitSpanChecker) checkBalanceTraffic( zap.String("changefeed", s.changefeedID.String()), zap.Int64("group", s.groupID), zap.Any("moveSpans", moveSpans), - zap.Any("minTrafficNodeID", minTrafficNodeID), + zap.Any("targetNodeID", targetNodeID), ) results = append(results, SplitSpanCheckResult{ OpType: OpMove, MoveSpans: moveSpans, - TargetNode: minTrafficNodeID, + TargetNode: targetNodeID, }) s.balanceCondition.reset() return @@ -1219,7 +1510,7 @@ func (s *SplitSpanChecker) checkBalanceTraffic( zap.Stringer("changefeed", s.changefeedID), zap.String("splitSpan", span.ID.String()), zap.Int64("group", s.groupID), - zap.Any("splitTargetNodes", []node.ID{minTrafficNodeID, maxTrafficNodeID}), + zap.Any("splitTargetNodes", []node.ID{targetNodeID, maxTrafficNodeID}), ) results = append(results, SplitSpanCheckResult{ @@ -1227,7 +1518,7 @@ func (s *SplitSpanChecker) checkBalanceTraffic( SplitSpan: span.SpanReplication, SpanNum: 2, SpanType: split.GetSplitType(span.regionCount), - SplitTargetNodes: []node.ID{minTrafficNodeID, maxTrafficNodeID}, // split the span, and one in minTrafficNode, one in maxTrafficNode, to balance traffic + SplitTargetNodes: []node.ID{targetNodeID, maxTrafficNodeID}, // split the span, and one in targetNode, one in maxTrafficNode, to balance traffic }) s.balanceCondition.reset() diff --git a/maintainer/replica/split_span_checker_test.go b/maintainer/replica/split_span_checker_test.go index a9efdeb6ce..fba3ce19e0 100644 --- a/maintainer/replica/split_span_checker_test.go +++ b/maintainer/replica/split_span_checker_test.go @@ -90,10 +90,29 @@ func createTestSplitSpanReplications(cfID common.ChangeFeedID, tableID int64, sp func newTestSplitChecker(t *testing.T, cfID common.ChangeFeedID, groupID pkgreplica.GroupID, schedulerCfg *config.ChangefeedSchedulerConfig) *SplitSpanChecker { refresher := NewRegionCountRefresher(cfID, util.GetOrZero(schedulerCfg.RegionCountRefreshInterval)) - checker := NewSplitSpanChecker(cfID, groupID, schedulerCfg, refresher) + resourceUsageTracker := NewNodeResourceUsageTracker() + resourceUsageTracker.ReplaceEventStoreWriteBytesPerSecond(nil, heartbeatpb.NodeResourceUsageStatus_UNSUPPORTED) + checker := NewSplitSpanChecker( + cfID, groupID, schedulerCfg, refresher, + resourceUsageTracker, &eventStoreBalanceLimiter{}) return checker } +func setEventStoreWriteBytesPerSecond( + tracker *NodeResourceUsageTracker, + rates map[node.ID]uint64, +) { + usages := make([]*heartbeatpb.NodeResourceUsage, 0, len(rates)) + for nodeID, rate := range rates { + usages = append(usages, &heartbeatpb.NodeResourceUsage{ + NodeId: nodeID.String(), + EventStoreWriteBytesPerSecond: rate, + }) + } + tracker.ReplaceEventStoreWriteBytesPerSecond( + usages, heartbeatpb.NodeResourceUsageStatus_AVAILABLE) +} + func TestSplitTableSpanIntoMultiple_Properties(t *testing.T) { spanA := common.TableIDToComparableSpan(common.DefaultKeyspaceID, 100) count := 10000 @@ -679,6 +698,523 @@ func TestSplitSpanChecker_CheckBalanceTraffic_Balance(t *testing.T) { require.True(t, moveResult.MoveSpans[0] == spanStatus2.SpanReplication) } +func TestSplitSpanChecker_CheckBalanceTraffic_AvoidsBusyEventStore(t *testing.T) { + testutil.SetUpTestServices(t) + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + + schedulerCfg := &config.ChangefeedSchedulerConfig{ + WriteKeyThreshold: util.AddressOf(1000), + RegionThreshold: util.AddressOf(10), + RegionCountRefreshInterval: util.AddressOf(time.Minute), + BalanceScoreThreshold: util.AddressOf(1), + MinTrafficPercentage: util.AddressOf(0.8), + MaxTrafficPercentage: util.AddressOf(1.2), + } + + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + for _, nodeID := range []node.ID{"source", "busy", "idle"} { + nodeManager.GetAliveNodes()[nodeID] = node.NewInfo(nodeID.String(), "") + } + + replicas := createTestSplitSpanReplications(cfID, 100000, 4) + checker := newTestSplitChecker(t, cfID, replicas[0].GetGroupID(), schedulerCfg) + for _, replica := range replicas { + checker.AddReplica(replica) + } + + // Group-local output traffic makes "busy" look like the best destination. + // The source has a small movable span and one large span. + replicas[0].SetNodeID("source") + replicas[1].SetNodeID("source") + replicas[2].SetNodeID("busy") + replicas[3].SetNodeID("idle") + traffic := []float64{300, 1300, 100, 300} + for i, replica := range replicas { + status := checker.allTasks[replica.ID] + status.lastThreeTraffic = []float64{traffic[i], traffic[i], traffic[i]} + status.regionCount = 3 + status.GetStatus().CheckpointTs = oracle.ComposeTS(time.Now().Add(-10*time.Second).UnixMilli(), 0) + } + checker.balanceCondition.statusUpdated = true + + // During the latest interval, the group-local minimum receives much more + // EventStore input from other work, while the third node has headroom. + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, + map[node.ID]uint64{"source": 300, "busy": 4000, "idle": 100}) + + results := checker.Check(10) + require.Len(t, results, 1) + moveResult := results.([]SplitSpanCheckResult)[0] + require.Equal(t, OpMove, moveResult.OpType) + require.Equal(t, node.ID("idle"), moveResult.TargetNode) + require.Equal(t, replicas[0], moveResult.MoveSpans[0]) +} + +func TestSplitSpanChecker_CheckBalanceEventStore_EvictsPersistentlyBusyNode(t *testing.T) { + testutil.SetUpTestServices(t) + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + + schedulerCfg := &config.ChangefeedSchedulerConfig{ + WriteKeyThreshold: util.AddressOf(1000), + RegionThreshold: util.AddressOf(10), + RegionCountRefreshInterval: util.AddressOf(time.Minute), + BalanceScoreThreshold: util.AddressOf(3), + MinTrafficPercentage: util.AddressOf(0.8), + MaxTrafficPercentage: util.AddressOf(1.2), + } + + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + for _, nodeID := range []node.ID{"busy", "idle"} { + nodeManager.GetAliveNodes()[nodeID] = node.NewInfo(nodeID.String(), "") + } + + replicas := createTestSplitSpanReplications(cfID, 100000, 4) + checker := newTestSplitChecker(t, cfID, replicas[0].GetGroupID(), schedulerCfg) + for _, replica := range replicas { + checker.AddReplica(replica) + } + + // The group-local traffic is balanced and too low to trigger the original + // traffic balance path. EventStore pressure alone must evict one dispatcher. + traffic := []float64{0.2, 0.1, 0.2, 0.1} + for i, replica := range replicas { + if i < 2 { + replica.SetNodeID("busy") + } else { + replica.SetNodeID("idle") + } + status := checker.allTasks[replica.ID] + status.lastThreeTraffic = []float64{traffic[i], traffic[i], traffic[i]} + status.regionCount = 3 + status.GetStatus().CheckpointTs = oracle.ComposeTS( + time.Now().Add(-10*time.Second).UnixMilli(), 0) + } + // A large relative skew at low absolute throughput is not evidence that the + // EventStore is overloaded. + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, + map[node.ID]uint64{"busy": 4000, "idle": 100}) + require.Empty(t, checker.Check(10)) + + overloadedRates := map[node.ID]uint64{ + "busy": 4 * minEventStoreWriteBytesPerSecond, + "idle": 100, + } + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, overloadedRates) + require.Empty(t, checker.Check(10)) + require.Equal(t, 1, checker.eventStoreBalanceCondition.balanceScore) + + // Dispatcher heartbeats and repeated checks must not count the same resource + // snapshot more than once. + for range 3 { + checker.eventStoreBalanceCondition.statusUpdated = true + require.Empty(t, checker.Check(10)) + require.Equal(t, 1, checker.eventStoreBalanceCondition.balanceScore) + } + + // A normal sample breaks the consecutive-overload sequence, so a transient + // spike cannot trigger a move. + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, + map[node.ID]uint64{"busy": 100, "idle": 100}) + require.Empty(t, checker.Check(10)) + require.Zero(t, checker.eventStoreBalanceCondition.balanceScore) + + // Only three distinct, consecutive overload samples reach the threshold. + for range 2 { + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, overloadedRates) + require.Empty(t, checker.Check(10)) + } + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, overloadedRates) + results := checker.Check(10) + require.Len(t, results, 1) + moveResult := results.([]SplitSpanCheckResult)[0] + require.Equal(t, OpMove, moveResult.OpType) + require.Equal(t, node.ID("idle"), moveResult.TargetNode) + require.Equal(t, replicas[0], moveResult.MoveSpans[0]) + + // Once this group has no dispatcher on the busy node, persistent EventStore + // pressure must not move a dispatcher back or emit another operation. + replicas[0].SetNodeID("idle") + replicas[1].SetNodeID("idle") + for i, replica := range replicas { + traffic := float64(100 + i*100) + checker.allTasks[replica.ID].lastThreeTraffic = []float64{traffic, traffic, traffic} + } + checker.balanceCondition.statusUpdated = true + checker.eventStoreBalanceCondition.statusUpdated = true + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, + map[node.ID]uint64{"busy": 4000, "idle": 100}) + require.Empty(t, checker.Check(10)) +} + +func TestSplitSpanChecker_CheckBalanceEventStoreUsesShortFollowUpWindow(t *testing.T) { + testutil.SetUpTestServices(t) + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + schedulerCfg := &config.ChangefeedSchedulerConfig{ + WriteKeyThreshold: util.AddressOf(100 * 1024 * 1024), + RegionThreshold: util.AddressOf(10), + RegionCountRefreshInterval: util.AddressOf(time.Minute), + BalanceScoreThreshold: util.AddressOf(5), + MinTrafficPercentage: util.AddressOf(0.8), + MaxTrafficPercentage: util.AddressOf(1.2), + } + + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + for _, nodeID := range []node.ID{"busy", "idle"} { + nodeManager.GetAliveNodes()[nodeID] = node.NewInfo(nodeID.String(), "") + } + + replicas := createTestSplitSpanReplications(cfID, 100000, 4) + checker := newTestSplitChecker(t, cfID, replicas[0].GetGroupID(), schedulerCfg) + for i, replica := range replicas { + if i < 3 { + replica.SetNodeID("busy") + } else { + replica.SetNodeID("idle") + } + checker.AddReplica(replica) + status := checker.allTasks[replica.ID] + status.lastThreeTraffic = []float64{1024 * 1024, 1024 * 1024, 1024 * 1024} + status.regionCount = 3 + } + + overloadedRates := map[node.ID]uint64{ + "busy": 4 * minEventStoreWriteBytesPerSecond, + "idle": minEventStoreWriteBytesPerSecond, + } + checkOverload := func() pkgreplica.GroupCheckResult { + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, overloadedRates) + return checker.Check(10) + } + + // The initial evacuation still requires the configured five overload + // observations. + for range 4 { + require.Empty(t, checkOverload()) + } + results := checkOverload() + require.Len(t, results, 1) + firstMove := results.([]SplitSpanCheckResult)[0] + require.Len(t, firstMove.MoveSpans, 1) + firstMove.MoveSpans[0].SetNodeID("idle") + + // Once evacuation has started, three fresh samples that still show overload + // are enough to move the next dispatcher. + for range 2 { + require.Empty(t, checkOverload()) + } + results = checkOverload() + require.Len(t, results, 1) + + // Recovery ends evacuation. A later overload must use the configured initial + // threshold again instead of the three-sample follow-up threshold. + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, + map[node.ID]uint64{"busy": 100, "idle": 100}) + require.Empty(t, checker.Check(10)) + require.Empty(t, checker.eventStoreBalanceLimiter.evacuatingSourceNodeID) + + for range 3 { + require.Empty(t, checkOverload()) + } + require.Equal(t, 3, checker.eventStoreBalanceCondition.balanceScore) +} + +func TestSplitSpanChecker_CheckBalanceEventStoreRequiresSameBusyNode(t *testing.T) { + testutil.SetUpTestServices(t) + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + schedulerCfg := &config.ChangefeedSchedulerConfig{ + WriteKeyThreshold: util.AddressOf(10 * 1024 * 1024), + RegionThreshold: util.AddressOf(10), + RegionCountRefreshInterval: util.AddressOf(time.Minute), + BalanceScoreThreshold: util.AddressOf(3), + MinTrafficPercentage: util.AddressOf(0.8), + MaxTrafficPercentage: util.AddressOf(1.2), + } + + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + for _, nodeID := range []node.ID{"node-a", "node-b", "idle"} { + nodeManager.GetAliveNodes()[nodeID] = node.NewInfo(nodeID.String(), "") + } + + replicas := createTestSplitSpanReplications(cfID, 100000, 3) + checker := newTestSplitChecker(t, cfID, replicas[0].GetGroupID(), schedulerCfg) + for i, replica := range replicas { + replica.SetNodeID([]node.ID{"node-a", "node-b", "idle"}[i]) + checker.AddReplica(replica) + status := checker.allTasks[replica.ID] + status.lastThreeTraffic = []float64{1024 * 1024, 1024 * 1024, 1024 * 1024} + status.regionCount = 3 + status.GetStatus().CheckpointTs = oracle.ComposeTS( + time.Now().Add(-10*time.Second).UnixMilli(), 0) + } + + setRates := func(busyNode node.ID) { + rates := map[node.ID]uint64{ + "node-a": 5 * 1024 * 1024, + "node-b": 5 * 1024 * 1024, + "idle": 100, + } + rates[busyNode] = 4 * minEventStoreWriteBytesPerSecond + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, rates) + } + + // Alternating spikes share the same idle destination but must not be treated + // as persistent overload on either source node. + for _, busyNode := range []node.ID{"node-a", "node-b", "node-a"} { + setRates(busyNode) + require.Empty(t, checker.Check(10)) + require.Equal(t, 1, checker.eventStoreBalanceCondition.balanceScore) + } + + // Two more samples for node-a make three consecutive observations and allow + // one dispatcher to move away from node-a. + setRates("node-a") + require.Empty(t, checker.Check(10)) + setRates("node-a") + results := checker.Check(10) + require.Len(t, results, 1) + moveResult := results.([]SplitSpanCheckResult)[0] + require.Equal(t, replicas[0], moveResult.MoveSpans[0]) + require.Equal(t, node.ID("node-b"), moveResult.TargetNode) +} + +func TestSplitSpanChecker_CheckBalanceEventStoreLimitsMovesAcrossGroups(t *testing.T) { + testutil.SetUpTestServices(t) + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + schedulerCfg := &config.ChangefeedSchedulerConfig{ + EnableTableAcrossNodes: util.AddressOf(true), + WriteKeyThreshold: util.AddressOf(10 * 1024 * 1024), + RegionThreshold: util.AddressOf(10), + RegionCountRefreshInterval: util.AddressOf(time.Minute), + BalanceScoreThreshold: util.AddressOf(1), + MinTrafficPercentage: util.AddressOf(0.8), + MaxTrafficPercentage: util.AddressOf(1.2), + } + + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + for _, nodeID := range []node.ID{"busy", "idle"} { + nodeManager.GetAliveNodes()[nodeID] = node.NewInfo(nodeID.String(), "") + } + + tracker := NewNodeResourceUsageTracker() + refresher := NewRegionCountRefresher(cfID, util.GetOrZero(schedulerCfg.RegionCountRefreshInterval)) + newChecker := GetNewGroupChecker(cfID, schedulerCfg, refresher, tracker) + checkers := make([]*SplitSpanChecker, 0, 2) + for tableID := int64(100000); tableID < 100002; tableID++ { + replicas := createTestSplitSpanReplications(cfID, tableID, 2) + checker := newChecker(replicas[0].GetGroupID()).(*SplitSpanChecker) + for i, replica := range replicas { + if i == 0 { + replica.SetNodeID("busy") + } else { + replica.SetNodeID("idle") + } + checker.AddReplica(replica) + status := checker.allTasks[replica.ID] + traffic := 0.1 + if tableID == 100001 && i == 0 { + traffic = 2 * 1024 * 1024 + } + status.lastThreeTraffic = []float64{traffic, traffic, traffic} + status.regionCount = 3 + status.GetStatus().CheckpointTs = oracle.ComposeTS( + time.Now().Add(-10*time.Second).UnixMilli(), 0) + } + checker.balanceCondition.statusUpdated = true + checker.eventStoreBalanceCondition.statusUpdated = true + checkers = append(checkers, checker) + } + require.Same(t, checkers[0].eventStoreBalanceLimiter, checkers[1].eventStoreBalanceLimiter) + + setEventStoreWriteBytesPerSecond(tracker, + map[node.ID]uint64{ + "busy": 4 * minEventStoreWriteBytesPerSecond, + "idle": 100, + }) + require.Len(t, checkers[0].Check(10), 1) + + // The second group also has traffic imbalance that would normally emit a + // split operation. Only one EventStore move is allowed for one resource + // snapshot, and the checker must not fall through to later scheduling paths. + require.Empty(t, checkers[1].Check(10)) + require.False(t, checkers[1].balanceCondition.statusUpdated) + checkers[1].eventStoreBalanceCondition.statusUpdated = true + require.Empty(t, checkers[1].Check(10)) + + setEventStoreWriteBytesPerSecond(tracker, + map[node.ID]uint64{ + "busy": 4 * minEventStoreWriteBytesPerSecond, + "idle": 100, + }) + require.Len(t, checkers[1].Check(10), 1) +} + +func TestSplitSpanChecker_MergePreparationRejectsBusyEventStoreTarget(t *testing.T) { + testutil.SetUpTestServices(t) + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + schedulerCfg := &config.ChangefeedSchedulerConfig{ + WriteKeyThreshold: util.AddressOf(1000), + RegionThreshold: util.AddressOf(20), + RegionCountRefreshInterval: util.AddressOf(time.Minute), + BalanceScoreThreshold: util.AddressOf(1), + MinTrafficPercentage: util.AddressOf(0.8), + MaxTrafficPercentage: util.AddressOf(1.2), + } + + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + for _, nodeID := range []node.ID{"overloaded", "node-b", "node-c"} { + nodeManager.GetAliveNodes()[nodeID] = node.NewInfo(nodeID.String(), "") + } + + replicas := createTestSplitSpanReplications(cfID, 100000, 12) + checker := newTestSplitChecker(t, cfID, replicas[0].GetGroupID(), schedulerCfg) + for i, replica := range replicas { + if i%2 == 0 { + replica.SetNodeID("node-b") + } else { + replica.SetNodeID("node-c") + } + checker.AddReplica(replica) + status := checker.allTasks[replica.ID] + status.lastThreeTraffic = []float64{1, 1, 1} + status.regionCount = 1 + status.GetStatus().CheckpointTs = oracle.ComposeTS( + time.Now().Add(-10*time.Second).UnixMilli(), 0) + } + checker.balanceCondition.statusUpdated = true + checker.eventStoreBalanceCondition.statusUpdated = true + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, + map[node.ID]uint64{ + "overloaded": 4 * minEventStoreWriteBytesPerSecond, + "node-b": minEventStoreWriteBytesPerSecond, + "node-c": minEventStoreWriteBytesPerSecond, + }) + + // Traffic balance and merge preparation both see the empty overloaded node + // as the group-local minimum, but neither may move a dispatcher onto it. + require.Empty(t, checker.Check(10)) +} + +func TestSplitSpanChecker_CheckBalanceEventStoreChoosesImprovingSpan(t *testing.T) { + testutil.SetUpTestServices(t) + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + schedulerCfg := &config.ChangefeedSchedulerConfig{ + WriteKeyThreshold: util.AddressOf(200 * 1024 * 1024), + RegionThreshold: util.AddressOf(20), + RegionCountRefreshInterval: util.AddressOf(time.Minute), + BalanceScoreThreshold: util.AddressOf(1), + MinTrafficPercentage: util.AddressOf(0.7), + MaxTrafficPercentage: util.AddressOf(1.3), + } + + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + for _, nodeID := range []node.ID{"node-a", "node-b"} { + nodeManager.GetAliveNodes()[nodeID] = node.NewInfo(nodeID.String(), "") + } + + replicas := createTestSplitSpanReplications(cfID, 100000, 3) + checker := newTestSplitChecker(t, cfID, replicas[0].GetGroupID(), schedulerCfg) + traffic := []float64{100 * 1024 * 1024, 10 * 1024 * 1024, 50 * 1024 * 1024} + for i, replica := range replicas { + if i < 2 { + replica.SetNodeID("node-a") + } else { + replica.SetNodeID("node-b") + } + checker.AddReplica(replica) + status := checker.allTasks[replica.ID] + status.lastThreeTraffic = []float64{traffic[i], traffic[i], traffic[i]} + status.regionCount = 3 + status.GetStatus().CheckpointTs = oracle.ComposeTS( + time.Now().Add(-10*time.Second).UnixMilli(), 0) + } + checker.eventStoreBalanceCondition.statusUpdated = true + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, + map[node.ID]uint64{"node-a": 110 * 1024 * 1024, "node-b": 50 * 1024 * 1024}) + + results := checker.Check(10) + require.Len(t, results, 1) + moveResult := results.([]SplitSpanCheckResult)[0] + require.Equal(t, replicas[1], moveResult.MoveSpans[0]) + require.Equal(t, node.ID("node-b"), moveResult.TargetNode) + + // Moving the 10 MiB/s span produces 100/60 MiB/s, which is inside the + // configured 1.3x boundary. The 100 MiB/s span would produce 10/150 and + // immediately invite the reverse move. + replicas[1].SetNodeID("node-b") + checker.eventStoreBalanceCondition.statusUpdated = true + checker.balanceCondition.statusUpdated = true + setEventStoreWriteBytesPerSecond(checker.nodeResourceUsage, + map[node.ID]uint64{"node-a": 100 * 1024 * 1024, "node-b": 60 * 1024 * 1024}) + require.Empty(t, checker.Check(10)) +} + +func TestSplitSpanChecker_CheckBalanceTraffic_RejectsUnsafeDestination(t *testing.T) { + tests := []struct { + name string + prepare func(*NodeResourceUsageTracker) + }{ + { + name: "destination busier than source", + prepare: func(tracker *NodeResourceUsageTracker) { + setEventStoreWriteBytesPerSecond(tracker, + map[node.ID]uint64{"source": 100, "destination": 4000}) + }, + }, + { + name: "supported telemetry becomes stale", + prepare: func(tracker *NodeResourceUsageTracker) { + now := time.Unix(100, 0) + tracker.now = func() time.Time { return now } + setEventStoreWriteBytesPerSecond(tracker, + map[node.ID]uint64{"source": 100, "destination": 10}) + now = now.Add(nodeResourceUsageStaleThreshold + time.Nanosecond) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testutil.SetUpTestServices(t) + cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) + schedulerCfg := &config.ChangefeedSchedulerConfig{ + WriteKeyThreshold: util.AddressOf(1000), + RegionThreshold: util.AddressOf(10), + RegionCountRefreshInterval: util.AddressOf(time.Minute), + BalanceScoreThreshold: util.AddressOf(1), + MinTrafficPercentage: util.AddressOf(0.8), + MaxTrafficPercentage: util.AddressOf(1.2), + } + + nodeManager := appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName) + for _, nodeID := range []node.ID{"source", "destination"} { + nodeManager.GetAliveNodes()[nodeID] = node.NewInfo(nodeID.String(), "") + } + + replicas := createTestSplitSpanReplications(cfID, 100000, 3) + checker := newTestSplitChecker(t, cfID, replicas[0].GetGroupID(), schedulerCfg) + for _, replica := range replicas { + checker.AddReplica(replica) + } + + replicas[0].SetNodeID("source") + replicas[1].SetNodeID("source") + replicas[2].SetNodeID("destination") + traffic := []float64{300, 1300, 100} + for i, replica := range replicas { + status := checker.allTasks[replica.ID] + status.lastThreeTraffic = []float64{traffic[i], traffic[i], traffic[i]} + status.regionCount = 3 + status.GetStatus().CheckpointTs = oracle.ComposeTS( + time.Now().Add(-10*time.Second).UnixMilli(), 0) + } + checker.balanceCondition.statusUpdated = true + tt.prepare(checker.nodeResourceUsage) + + require.Empty(t, checker.Check(10)) + }) + } +} + func TestSplitSpanChecker_CheckBalanceTraffic_NoBalanceNeeded(t *testing.T) { testutil.SetUpTestServices(t) cfID := common.NewChangeFeedIDWithName("test", common.DefaultKeyspaceName) diff --git a/maintainer/scheduler/balance_splits.go b/maintainer/scheduler/balance_splits.go index 7898fdf5de..38c2bf84cf 100644 --- a/maintainer/scheduler/balance_splits.go +++ b/maintainer/scheduler/balance_splits.go @@ -99,14 +99,15 @@ func (s *balanceSplitsScheduler) Execute() time.Time { // // 1. First check if there are existing spans that need to be split (spans' region count exceeds the limit, span's traffic exceeds the limit) // if so, split first - // 2. Check if the current spans meet the requirement to merge all back into one span + // 2. Move one dispatcher away from a node whose EventStore traffic is persistently above the cluster average + // 3. Check if the current spans meet the requirement to merge all back into one span // (total region count and traffic are within limits, and lag is low), if so, merge (merge is done by first move, then merge) - // 3. Check if traffic is balanced between nodes, if there is obvious imbalance, + // 4. Check if traffic is balanced between nodes, if there is obvious imbalance, // first try to migrate spans from the node with maximum traffic to the node with minimum traffic, // if not possible, split the span of the maximum traffic node and move it over - // 4. Check if there are dispatchers that need to be merged, satisfying that they are adjacent in the same node, lag is low, + // 5. Check if there are dispatchers that need to be merged, satisfying that they are adjacent in the same node, lag is low, // and the merged span's region count and traffic are within limits - // 5. If none of the above, first calculate whether the current number of dispatchers is within acceptable range, + // 6. If none of the above, first calculate whether the current number of dispatchers is within acceptable range, // if not, check what dispatchers can be moved to facilitate merge operations. // we only process spans of one group that are all in replicating state (merge/split/move operations will enter scheduling state once created) diff --git a/maintainer/scheduler/drain_test.go b/maintainer/scheduler/drain_test.go index 8e6f246f61..3a97a46134 100644 --- a/maintainer/scheduler/drain_test.go +++ b/maintainer/scheduler/drain_test.go @@ -314,7 +314,7 @@ func newDrainSchedulerTestHarness( self, false, ) - sc := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + sc := span.NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) oc := operator.NewOperatorController(cfID, sc, 100, common.DefaultMode) return cfID, nodeManager, oc, sc, NewDrainState(), self } diff --git a/maintainer/span/span_controller.go b/maintainer/span/span_controller.go index 93f54289b7..f401c7c9f0 100644 --- a/maintainer/span/span_controller.go +++ b/maintainer/span/span_controller.go @@ -97,11 +97,12 @@ func NewController( refresher *replica.RegionCountRefresher, keyspaceID uint32, mode int64, + nodeResourceUsage *replica.NodeResourceUsageTracker, ) *Controller { c := &Controller{ changefeedID: changefeedID, ddlSpan: ddlSpan, - newGroupChecker: replica.GetNewGroupChecker(changefeedID, schedulerCfg, refresher), + newGroupChecker: replica.GetNewGroupChecker(changefeedID, schedulerCfg, refresher, nodeResourceUsage), nodeManager: appcontext.GetService[*watcher.NodeManager](watcher.NodeManagerName), splitter: splitter, ddlDispatcherID: ddlSpan.ID, diff --git a/maintainer/span/span_controller_test.go b/maintainer/span/span_controller_test.go index e99ebf74a1..4a5b06d98a 100644 --- a/maintainer/span/span_controller_test.go +++ b/maintainer/span/span_controller_test.go @@ -42,7 +42,7 @@ func newControllerForCheckpointTsTrackerTest(t *testing.T) *Controller { CheckpointTs: 1, }, "node1", false) appcontext.SetService(watcher.NodeManagerName, watcher.NewNodeManager(nil, nil)) - return NewController(changefeedID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + return NewController(changefeedID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) } func newSpanReplicationForCheckpointTsTrackerTest( @@ -206,7 +206,7 @@ func TestNewController(t *testing.T) { CheckpointTs: 1, }, "node1", false) appcontext.SetService(watcher.NodeManagerName, watcher.NewNodeManager(nil, nil)) - controller := NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode) + controller := NewController(cfID, ddlSpan, nil, nil, nil, common.DefaultKeyspaceID, common.DefaultMode, replica.NewNodeResourceUsageTracker()) require.NotNil(t, controller) require.Equal(t, cfID, controller.changefeedID) require.False(t, controller.enableTableAcrossNodes) @@ -231,6 +231,7 @@ func TestController_AddNewTable(t *testing.T) { nil, common.DefaultKeyspaceID, common.DefaultMode, + replica.NewNodeResourceUsageTracker(), ) table := commonEvent.Table{ @@ -269,6 +270,7 @@ func TestController_GetTaskByID(t *testing.T) { nil, common.DefaultKeyspaceID, common.DefaultMode, + replica.NewNodeResourceUsageTracker(), ) // Add a table first @@ -324,6 +326,7 @@ func TestController_GetTasksByTableID(t *testing.T) { nil, common.DefaultKeyspaceID, common.DefaultMode, + replica.NewNodeResourceUsageTracker(), ) // Add a table @@ -362,6 +365,7 @@ func TestController_GetTasksBySchemaID(t *testing.T) { nil, common.DefaultKeyspaceID, common.DefaultMode, + replica.NewNodeResourceUsageTracker(), ) // Add tables from the same schema @@ -404,6 +408,7 @@ func TestController_UpdateSchemaID(t *testing.T) { nil, common.DefaultKeyspaceID, common.DefaultMode, + replica.NewNodeResourceUsageTracker(), ) // Add a table @@ -447,6 +452,7 @@ func TestController_Statistics(t *testing.T) { nil, common.DefaultKeyspaceID, common.DefaultMode, + replica.NewNodeResourceUsageTracker(), ) // Add some tables @@ -679,5 +685,6 @@ func newControllerWithCheckerForTest(t *testing.T) *Controller { nil, common.DefaultKeyspaceID, common.DefaultMode, + replica.NewNodeResourceUsageTracker(), ) } diff --git a/pkg/config/scheduler_config.go b/pkg/config/scheduler_config.go index 89298f4b54..219069ea7d 100644 --- a/pkg/config/scheduler_config.go +++ b/pkg/config/scheduler_config.go @@ -60,10 +60,10 @@ type ChangefeedSchedulerConfig struct { // BalanceScoreThreshold is the score threshold for balancing traffic. Larger value means less frequent balancing. // Default value is 20 BalanceScoreThreshold *int `toml:"balance-score-threshold" json:"balance-score-threshold,omitempty"` - // MinTrafficPercentage is the minimum traffic percentage for balancing traffic. Larger value means less frequent balancing. + // MinTrafficPercentage is the minimum traffic percentage for balancing traffic. Larger value means more frequent balancing. // MinTrafficPercentage must be less then 1. Default value is 0.8 MinTrafficPercentage *float64 `toml:"min-traffic-percentage" json:"min-traffic-percentage,omitempty"` - // MaxTrafficPercentage is the maximum traffic percentage for balancing traffic. Less value means less frequent balancing. + // MaxTrafficPercentage is the maximum traffic percentage for balancing traffic. Smaller value means more frequent balancing. // MaxTrafficPercentage must be greater then 1. Default value is 1.25 MaxTrafficPercentage *float64 `toml:"max-traffic-percentage" json:"max-traffic-percentage,omitempty"` } diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index 445868e0ac..aacb202c4b 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -354,6 +354,10 @@ func (m *mockEventStore) GetLogCoordinatorNodeID() node.ID { return "" } +func (m *mockEventStore) EventStoreWriteBytes() uint64 { + return 0 +} + func (m *mockEventStore) RegisterDispatcher( changefeedID common.ChangeFeedID, dispatcherID common.DispatcherID, diff --git a/pkg/messaging/message_write_lease_test.go b/pkg/messaging/message_write_lease_test.go index c7946f7f29..43ca303cda 100644 --- a/pkg/messaging/message_write_lease_test.go +++ b/pkg/messaging/message_write_lease_test.go @@ -46,6 +46,11 @@ func TestNodeHeartbeatResponseRemoteRoundTrip(t *testing.T) { TargetNodeEpoch: 11, RequestSeq: 12, LeaseDurationMs: 5000, + NodeResourceUsages: []*heartbeatpb.NodeResourceUsage{ + {NodeId: "capture-1", EventStoreWriteBytes: 100}, + {NodeId: "capture-2", EventStoreWriteBytes: 200}, + }, + NodeResourceUsageStatus: heartbeatpb.NodeResourceUsageStatus_AVAILABLE, } require.NoError(t, sender.SendCommand( NewSingleTargetMessage(receiver.id, MaintainerManagerTopic, response), @@ -75,6 +80,10 @@ func TestNodeHeartbeatResponseIOTypeRoundTrip(t *testing.T) { WitnessNodeEpoch: 13, Nonce: []byte("nonce"), }, + NodeResourceUsages: []*heartbeatpb.NodeResourceUsage{ + {NodeId: "capture-1", EventStoreWriteBytes: 100}, + }, + NodeResourceUsageStatus: heartbeatpb.NodeResourceUsageStatus_AVAILABLE, } message := NewSingleTargetMessage(node.ID("capture"), MaintainerManagerTopic, response) require.Equal(t, TypeNodeHeartbeatResponse, message.Type) diff --git a/pkg/metrics/event_store.go b/pkg/metrics/event_store.go index f98463245d..88223055ea 100644 --- a/pkg/metrics/event_store.go +++ b/pkg/metrics/event_store.go @@ -13,9 +13,7 @@ package metrics -import ( - "github.com/prometheus/client_golang/prometheus" -) +import "github.com/prometheus/client_golang/prometheus" var ( EventStoreSubscriptionGauge = prometheus.NewGauge( diff --git a/pkg/metrics/scheduler.go b/pkg/metrics/scheduler.go index b17a1d291f..1d99ae96b4 100644 --- a/pkg/metrics/scheduler.go +++ b/pkg/metrics/scheduler.go @@ -144,6 +144,13 @@ var ( Help: "Bucketed histogram of split span check time (s).", Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms~524s }, []string{GetKeyspaceLabel(), "changefeed", "group_id"}) + TrafficBalanceSkipCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "maintainer", + Name: "traffic_balance_skip_total", + Help: "The number of traffic balance checks skipped for safety.", + }, []string{"reason"}) ) func initSchedulerMetrics(registry *prometheus.Registry) { @@ -168,4 +175,5 @@ func initSchedulerMetrics(registry *prometheus.Registry) { registry.MustRegister(SlowestTablePullerResolvedTsLag) registry.MustRegister(SplitSpanCheckDuration) + registry.MustRegister(TrafficBalanceSkipCounter) } diff --git a/server/server.go b/server/server.go index 27faa3bb8e..6076139422 100644 --- a/server/server.go +++ b/server/server.go @@ -242,7 +242,7 @@ func (c *server) initialize(ctx context.Context) error { subscriptionClient, schemaStore, eventStore, - maintainer.NewMaintainerManager(c.info, conf.Debug.Scheduler, &c.liveness), + maintainer.NewMaintainerManager(c.info, conf.Debug.Scheduler, &c.liveness, eventStore), eventService, } // register it into global var diff --git a/tests/integration_tests/run_light_it_in_ci.sh b/tests/integration_tests/run_light_it_in_ci.sh index 2b4e52702f..6eb3755ed0 100755 --- a/tests/integration_tests/run_light_it_in_ci.sh +++ b/tests/integration_tests/run_light_it_in_ci.sh @@ -48,7 +48,7 @@ mysql_groups=( # G07 'fail_over_ddl_H changefeed_update_config synced_status_with_redo redo_apply_table_route' # G08 - 'capture_session_done_during_task changefeed_dup_error_restart mysql_sink_retry fail_over_ddl_I table_route' + 'capture_session_done_during_task split_table_balance_event_store changefeed_dup_error_restart mysql_sink_retry fail_over_ddl_I table_route' # G09 'sequence cdc_server_tips ddl_sequence server_config_compatibility log_redaction fail_over_ddl_J' # G10 diff --git a/tests/integration_tests/split_table_balance_event_store/conf/changefeed.toml b/tests/integration_tests/split_table_balance_event_store/conf/changefeed.toml new file mode 100644 index 0000000000..71cf375d2d --- /dev/null +++ b/tests/integration_tests/split_table_balance_event_store/conf/changefeed.toml @@ -0,0 +1,6 @@ +[scheduler] +enable-table-across-nodes = true +region-threshold = 2 +region-count-per-span = 2 +write-key-threshold = 1073741824 +balance-score-threshold = 3 diff --git a/tests/integration_tests/split_table_balance_event_store/conf/diff_config.toml b/tests/integration_tests/split_table_balance_event_store/conf/diff_config.toml new file mode 100644 index 0000000000..eed2b15cb7 --- /dev/null +++ b/tests/integration_tests/split_table_balance_event_store/conf/diff_config.toml @@ -0,0 +1,22 @@ +check-thread-count = 4 +export-fix-sql = true +check-struct-only = false + +[task] +output-dir = "/tmp/tidb_cdc_test/split_table_balance_event_store/sync_diff/output" +source-instances = ["upstream"] +target-instance = "downstream" +target-check-tables = ["split_table_balance_event_store.*"] + +[data-sources] +[data-sources.upstream] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" + +[data-sources.downstream] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "" diff --git a/tests/integration_tests/split_table_balance_event_store/run.sh b/tests/integration_tests/split_table_balance_event_store/run.sh new file mode 100755 index 0000000000..325508698e --- /dev/null +++ b/tests/integration_tests/split_table_balance_event_store/run.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +set -eu + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $CUR/../_utils/test_prepare + +WORK_DIR=$OUT_DIR/$TEST_NAME +CDC_BINARY=cdc.test +SINK_TYPE=$1 + +DB_NAME=split_table_balance_event_store +TABLE_NAME=t +CHANGEFEED_ID=test +API_ADDR=127.0.0.1:8300 +SOURCE_ADDR=127.0.0.1:8301 +EVENT_STORE_FAILPOINT=github.com/pingcap/ticdc/logservice/eventstore/InjectEventStoreWriteBytes + +workload_pid="" +failpoint_enabled=false + +function cleanup() { + if [ -n "$workload_pid" ]; then + kill "$workload_pid" 2>/dev/null || true + wait "$workload_pid" 2>/dev/null || true + fi + if [ "$failpoint_enabled" == "true" ]; then + disable_failpoint --addr "$SOURCE_ADDR" --name "$EVENT_STORE_FAILPOINT" || true + fi + stop_test $WORK_DIR +} + +function get_capture_id() { + local addr=$1 + curl -fsS "http://${API_ADDR}/api/v2/captures" | + jq -r --arg addr "$addr" '.items[] | select(.address == $addr) | .id' | head -n1 +} + +function get_table_counts() { + local table_id=$1 + local source_id=$2 + curl -fsS "http://${API_ADDR}/api/v2/changefeeds/${CHANGEFEED_ID}/tables?keyspace=${KEYSPACE_NAME}" | + jq -r --argjson table_id "$table_id" --arg source_id "$source_id" ' + [.items[] | .node_id as $node_id | .table_ids[] | + select(. == $table_id) | {node_id: $node_id}] as $dispatchers | + [($dispatchers | length), + ([$dispatchers[] | select(.node_id == $source_id)] | length)] | + @tsv' +} + +function wait_for_split_table_on_source() { + local table_id=$1 + local source_id=$2 + for ((i = 0; i < 60; i++)); do + read -r total source < <(get_table_counts "$table_id" "$source_id") + if [ "$total" -ge 4 ] && [ "$source" -eq "$total" ]; then + echo "$total" + return 0 + fi + sleep 2 + done + echo "split table dispatchers were not all scheduled on $SOURCE_ADDR" >&2 + return 1 +} + +function wait_for_redistribution() { + local table_id=$1 + local source_id=$2 + local initial_count=$3 + for ((i = 0; i < 60; i++)); do + read -r total source < <(get_table_counts "$table_id" "$source_id") + if [ "$total" -eq "$initial_count" ] && [ "$source" -lt "$initial_count" ]; then + echo "dispatcher distribution changed: source=$source, other=$((total - source))" + return 0 + fi + sleep 2 + done + echo "dispatchers were not moved away from the EventStore-heavy node" >&2 + return 1 +} + +function generate_table_traffic() { + while true; do + mysql -h${UP_TIDB_HOST} -P${UP_TIDB_PORT} -uroot -N -s \ + -e "UPDATE ${DB_NAME}.${TABLE_NAME} SET payload=REPEAT(IF(LEFT(payload, 1)='x', 'y', 'x'), 1024), seq=seq+1;" \ + >/dev/null + sleep 0.2 + done +} + +function run() { + if [ "$SINK_TYPE" != "mysql" ]; then + return + fi + + rm -rf $WORK_DIR && mkdir -p $WORK_DIR + start_tidb_cluster --workdir $WORK_DIR + + local pd_addr=http://${UP_PD_HOST_1}:${UP_PD_PORT_1} + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --pd "$pd_addr" --logsuffix 0 --addr "$API_ADDR" + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --pd "$pd_addr" --logsuffix 1 --addr "$SOURCE_ADDR" + + run_sql "CREATE DATABASE ${DB_NAME};" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "CREATE DATABASE ${DB_NAME};" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + run_sql "CREATE TABLE ${DB_NAME}.${TABLE_NAME} (id INT PRIMARY KEY, seq BIGINT NOT NULL, payload MEDIUMTEXT NOT NULL);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "CREATE TABLE ${DB_NAME}.${TABLE_NAME} (id INT PRIMARY KEY, seq BIGINT NOT NULL, payload MEDIUMTEXT NOT NULL);" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + run_sql "INSERT INTO ${DB_NAME}.${TABLE_NAME} VALUES (1000,0,''),(11000,0,''),(21000,0,''),(31000,0,''),(41000,0,''),(51000,0,''),(61000,0,''),(71000,0,'');" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "INSERT INTO ${DB_NAME}.${TABLE_NAME} VALUES (1000,0,''),(11000,0,''),(21000,0,''),(31000,0,''),(41000,0,''),(51000,0,''),(61000,0,''),(71000,0,'');" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + run_sql "SPLIT TABLE ${DB_NAME}.${TABLE_NAME} BETWEEN (0) AND (80000) REGIONS 8;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + + local start_ts + start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) + do_retry 5 3 cdc_cli_changefeed create --pd="$pd_addr" --start-ts="$start_ts" \ + --sink-uri="mysql://normal:123456@127.0.0.1:3306/" -c "$CHANGEFEED_ID" \ + --config="$CUR/conf/changefeed.toml" + + local table_id + local source_id + local initial_count + table_id=$(get_table_id "$DB_NAME" "$TABLE_NAME") + split_table_with_retry "$table_id" "$CHANGEFEED_ID" 10 + move_split_table_with_retry "$SOURCE_ADDR" "$table_id" "$CHANGEFEED_ID" 10 + source_id=$(get_capture_id "$SOURCE_ADDR") + initial_count=$(wait_for_split_table_on_source "$table_id" "$source_id") + + # Keep dispatcher output below the normal 1 MiB/s traffic-balance threshold, + # while counting each EventStore write batch as 1 GiB on the source node. + # This makes EventStore pressure the only reason to move a dispatcher. + enable_failpoint --addr "$SOURCE_ADDR" --name "$EVENT_STORE_FAILPOINT" --expr "return(1073741824)" + failpoint_enabled=true + generate_table_traffic & + workload_pid=$! + + wait_for_redistribution "$table_id" "$source_id" "$initial_count" + + kill "$workload_pid" + wait "$workload_pid" 2>/dev/null || true + workload_pid="" + disable_failpoint --addr "$SOURCE_ADDR" --name "$EVENT_STORE_FAILPOINT" + failpoint_enabled=false + + check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml 60 + cleanup_process $CDC_BINARY +} + +trap cleanup EXIT +run "$@" +check_logs $WORK_DIR +echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"