From 9b1981ce4667bfe02c66ebf6731dbc9ccb8a65c2 Mon Sep 17 00:00:00 2001 From: David Reiss Date: Mon, 29 Jan 2024 13:27:44 -0800 Subject: [PATCH] Do schedule backfills incrementally (#5344) Previously schedule backfills were processed synchronously: the workflow would run through the whole given time range and either start workflows or add them to the buffer. This is changed to process them incrementally and keep track of the time range processed so far. There can be up to 1000 buffered backfills (reusing same limit as action buffer). - This allows backfills to be arbitrarily long, instead of limited to 1000 actions. - In some situations, evaluating a long backfill could take more than 1s of elapsed time, causing the SDK to think that the workflow was deadlocked and failing the workflow task, effectively causing the scheduler workflow to get stuck. new unit tests, replay test --- api/schedule/v1/message.pb.go | 437 +++++++++--------- .../server/api/schedule/v1/message.proto | 3 +- .../replay_with_incremental_backfill.json.gz | Bin 0 -> 12234 bytes service/worker/scheduler/workflow.go | 128 +++-- service/worker/scheduler/workflow_test.go | 156 ++++++- 5 files changed, 471 insertions(+), 253 deletions(-) create mode 100644 service/worker/scheduler/testdata/replay_with_incremental_backfill.json.gz diff --git a/api/schedule/v1/message.pb.go b/api/schedule/v1/message.pb.go index 4bc45327ef2..5161d25b61c 100644 --- a/api/schedule/v1/message.pb.go +++ b/api/schedule/v1/message.pb.go @@ -32,10 +32,10 @@ import ( reflect "reflect" sync "sync" - v11 "go.temporal.io/api/common/v1" + v12 "go.temporal.io/api/common/v1" v1 "go.temporal.io/api/enums/v1" - v12 "go.temporal.io/api/failure/v1" - v13 "go.temporal.io/api/schedule/v1" + v13 "go.temporal.io/api/failure/v1" + v11 "go.temporal.io/api/schedule/v1" v14 "go.temporal.io/api/workflowservice/v1" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" @@ -128,16 +128,15 @@ type InternalState struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` - NamespaceId string `protobuf:"bytes,2,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - // this is just the workflow id of the scheduler, but accessing that from the workflow is - // currently not available during replay, so we just pass it in here. + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + NamespaceId string `protobuf:"bytes,2,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` ScheduleId string `protobuf:"bytes,8,opt,name=schedule_id,json=scheduleId,proto3" json:"schedule_id,omitempty"` LastProcessedTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=last_processed_time,json=lastProcessedTime,proto3" json:"last_processed_time,omitempty"` BufferedStarts []*BufferedStart `protobuf:"bytes,4,rep,name=buffered_starts,json=bufferedStarts,proto3" json:"buffered_starts,omitempty"` + OngoingBackfills []*v11.BackfillRequest `protobuf:"bytes,10,rep,name=ongoing_backfills,json=ongoingBackfills,proto3" json:"ongoing_backfills,omitempty"` // last completion/failure - LastCompletionResult *v11.Payloads `protobuf:"bytes,5,opt,name=last_completion_result,json=lastCompletionResult,proto3" json:"last_completion_result,omitempty"` - ContinuedFailure *v12.Failure `protobuf:"bytes,6,opt,name=continued_failure,json=continuedFailure,proto3" json:"continued_failure,omitempty"` + LastCompletionResult *v12.Payloads `protobuf:"bytes,5,opt,name=last_completion_result,json=lastCompletionResult,proto3" json:"last_completion_result,omitempty"` + ContinuedFailure *v13.Failure `protobuf:"bytes,6,opt,name=continued_failure,json=continuedFailure,proto3" json:"continued_failure,omitempty"` // conflict token is implemented as simple sequence number ConflictToken int64 `protobuf:"varint,7,opt,name=conflict_token,json=conflictToken,proto3" json:"conflict_token,omitempty"` NeedRefresh bool `protobuf:"varint,9,opt,name=need_refresh,json=needRefresh,proto3" json:"need_refresh,omitempty"` @@ -210,14 +209,21 @@ func (x *InternalState) GetBufferedStarts() []*BufferedStart { return nil } -func (x *InternalState) GetLastCompletionResult() *v11.Payloads { +func (x *InternalState) GetOngoingBackfills() []*v11.BackfillRequest { + if x != nil { + return x.OngoingBackfills + } + return nil +} + +func (x *InternalState) GetLastCompletionResult() *v12.Payloads { if x != nil { return x.LastCompletionResult } return nil } -func (x *InternalState) GetContinuedFailure() *v12.Failure { +func (x *InternalState) GetContinuedFailure() *v13.Failure { if x != nil { return x.ContinuedFailure } @@ -243,9 +249,9 @@ type StartScheduleArgs struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Schedule *v13.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` - Info *v13.ScheduleInfo `protobuf:"bytes,2,opt,name=info,proto3" json:"info,omitempty"` - InitialPatch *v13.SchedulePatch `protobuf:"bytes,3,opt,name=initial_patch,json=initialPatch,proto3" json:"initial_patch,omitempty"` + Schedule *v11.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` + Info *v11.ScheduleInfo `protobuf:"bytes,2,opt,name=info,proto3" json:"info,omitempty"` + InitialPatch *v11.SchedulePatch `protobuf:"bytes,3,opt,name=initial_patch,json=initialPatch,proto3" json:"initial_patch,omitempty"` State *InternalState `protobuf:"bytes,4,opt,name=state,proto3" json:"state,omitempty"` } @@ -281,21 +287,21 @@ func (*StartScheduleArgs) Descriptor() ([]byte, []int) { return file_temporal_server_api_schedule_v1_message_proto_rawDescGZIP(), []int{2} } -func (x *StartScheduleArgs) GetSchedule() *v13.Schedule { +func (x *StartScheduleArgs) GetSchedule() *v11.Schedule { if x != nil { return x.Schedule } return nil } -func (x *StartScheduleArgs) GetInfo() *v13.ScheduleInfo { +func (x *StartScheduleArgs) GetInfo() *v11.ScheduleInfo { if x != nil { return x.Info } return nil } -func (x *StartScheduleArgs) GetInitialPatch() *v13.SchedulePatch { +func (x *StartScheduleArgs) GetInitialPatch() *v11.SchedulePatch { if x != nil { return x.InitialPatch } @@ -314,7 +320,7 @@ type FullUpdateRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Schedule *v13.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` + Schedule *v11.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` ConflictToken int64 `protobuf:"varint,2,opt,name=conflict_token,json=conflictToken,proto3" json:"conflict_token,omitempty"` } @@ -350,7 +356,7 @@ func (*FullUpdateRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_schedule_v1_message_proto_rawDescGZIP(), []int{3} } -func (x *FullUpdateRequest) GetSchedule() *v13.Schedule { +func (x *FullUpdateRequest) GetSchedule() *v11.Schedule { if x != nil { return x.Schedule } @@ -369,8 +375,8 @@ type DescribeResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Schedule *v13.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` - Info *v13.ScheduleInfo `protobuf:"bytes,2,opt,name=info,proto3" json:"info,omitempty"` + Schedule *v11.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` + Info *v11.ScheduleInfo `protobuf:"bytes,2,opt,name=info,proto3" json:"info,omitempty"` ConflictToken int64 `protobuf:"varint,3,opt,name=conflict_token,json=conflictToken,proto3" json:"conflict_token,omitempty"` } @@ -406,14 +412,14 @@ func (*DescribeResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_schedule_v1_message_proto_rawDescGZIP(), []int{4} } -func (x *DescribeResponse) GetSchedule() *v13.Schedule { +func (x *DescribeResponse) GetSchedule() *v11.Schedule { if x != nil { return x.Schedule } return nil } -func (x *DescribeResponse) GetInfo() *v13.ScheduleInfo { +func (x *DescribeResponse) GetInfo() *v11.ScheduleInfo { if x != nil { return x.Info } @@ -434,7 +440,7 @@ type WatchWorkflowRequest struct { // Note: this will be sent to the activity with empty execution.run_id, and // the run id that we started in first_execution_run_id. - Execution *v11.WorkflowExecution `protobuf:"bytes,3,opt,name=execution,proto3" json:"execution,omitempty"` + Execution *v12.WorkflowExecution `protobuf:"bytes,3,opt,name=execution,proto3" json:"execution,omitempty"` FirstExecutionRunId string `protobuf:"bytes,4,opt,name=first_execution_run_id,json=firstExecutionRunId,proto3" json:"first_execution_run_id,omitempty"` LongPoll bool `protobuf:"varint,5,opt,name=long_poll,json=longPoll,proto3" json:"long_poll,omitempty"` } @@ -471,7 +477,7 @@ func (*WatchWorkflowRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_schedule_v1_message_proto_rawDescGZIP(), []int{5} } -func (x *WatchWorkflowRequest) GetExecution() *v11.WorkflowExecution { +func (x *WatchWorkflowRequest) GetExecution() *v12.WorkflowExecution { if x != nil { return x.Execution } @@ -551,14 +557,14 @@ func (m *WatchWorkflowResponse) GetResultFailure() isWatchWorkflowResponse_Resul return nil } -func (x *WatchWorkflowResponse) GetResult() *v11.Payloads { +func (x *WatchWorkflowResponse) GetResult() *v12.Payloads { if x, ok := x.GetResultFailure().(*WatchWorkflowResponse_Result); ok { return x.Result } return nil } -func (x *WatchWorkflowResponse) GetFailure() *v12.Failure { +func (x *WatchWorkflowResponse) GetFailure() *v13.Failure { if x, ok := x.GetResultFailure().(*WatchWorkflowResponse_Failure); ok { return x.Failure } @@ -570,11 +576,11 @@ type isWatchWorkflowResponse_ResultFailure interface { } type WatchWorkflowResponse_Result struct { - Result *v11.Payloads `protobuf:"bytes,2,opt,name=result,proto3,oneof"` + Result *v12.Payloads `protobuf:"bytes,2,opt,name=result,proto3,oneof"` } type WatchWorkflowResponse_Failure struct { - Failure *v12.Failure `protobuf:"bytes,3,opt,name=failure,proto3,oneof"` + Failure *v13.Failure `protobuf:"bytes,3,opt,name=failure,proto3,oneof"` } func (*WatchWorkflowResponse_Result) isWatchWorkflowResponse_ResultFailure() {} @@ -699,7 +705,7 @@ type CancelWorkflowRequest struct { RequestId string `protobuf:"bytes,3,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` Identity string `protobuf:"bytes,4,opt,name=identity,proto3" json:"identity,omitempty"` // Note: run id in execution is first execution run id - Execution *v11.WorkflowExecution `protobuf:"bytes,5,opt,name=execution,proto3" json:"execution,omitempty"` + Execution *v12.WorkflowExecution `protobuf:"bytes,5,opt,name=execution,proto3" json:"execution,omitempty"` Reason string `protobuf:"bytes,6,opt,name=reason,proto3" json:"reason,omitempty"` } @@ -749,7 +755,7 @@ func (x *CancelWorkflowRequest) GetIdentity() string { return "" } -func (x *CancelWorkflowRequest) GetExecution() *v11.WorkflowExecution { +func (x *CancelWorkflowRequest) GetExecution() *v12.WorkflowExecution { if x != nil { return x.Execution } @@ -771,7 +777,7 @@ type TerminateWorkflowRequest struct { RequestId string `protobuf:"bytes,3,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` Identity string `protobuf:"bytes,4,opt,name=identity,proto3" json:"identity,omitempty"` // Note: run id in execution is first execution run id - Execution *v11.WorkflowExecution `protobuf:"bytes,5,opt,name=execution,proto3" json:"execution,omitempty"` + Execution *v12.WorkflowExecution `protobuf:"bytes,5,opt,name=execution,proto3" json:"execution,omitempty"` Reason string `protobuf:"bytes,6,opt,name=reason,proto3" json:"reason,omitempty"` } @@ -821,7 +827,7 @@ func (x *TerminateWorkflowRequest) GetIdentity() string { return "" } -func (x *TerminateWorkflowRequest) GetExecution() *v11.WorkflowExecution { +func (x *TerminateWorkflowRequest) GetExecution() *v12.WorkflowExecution { if x != nil { return x.Execution } @@ -937,8 +943,8 @@ var file_temporal_server_api_schedule_v1_message_proto_rawDesc = []byte{ 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x24, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x65, 0x6e, 0x75, 0x6d, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x26, 0x74, 0x65, 0x6d, - 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, - 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, + 0x6c, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x36, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, @@ -952,25 +958,25 @@ var file_temporal_server_api_schedule_v1_message_proto_rawDesc = []byte{ 0x6d, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x3f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x42, - 0x02, 0x68, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x6f, 0x76, 0x65, 0x72, 0x6c, 0x61, 0x70, 0x5f, 0x70, 0x6f, - 0x6c, 0x69, 0x63, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x74, 0x65, 0x6d, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, + 0x42, 0x02, 0x68, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x6f, 0x76, 0x65, 0x72, 0x6c, 0x61, 0x70, 0x5f, 0x70, + 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x65, 0x6e, 0x75, 0x6d, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x4f, 0x76, 0x65, 0x72, 0x6c, 0x61, 0x70, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x0d, 0x6f, 0x76, 0x65, 0x72, 0x6c, 0x61, 0x70, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x42, 0x02, 0x68, 0x00, 0x12, 0x1a, 0x0a, 0x06, 0x6d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x6d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x42, 0x02, - 0x68, 0x00, 0x22, 0xab, 0x04, 0x0a, 0x0d, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x74, + 0x68, 0x00, 0x22, 0x87, 0x05, 0x0a, 0x0d, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x20, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x25, 0x0a, 0x0c, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x42, 0x02, 0x68, 0x00, 0x12, 0x23, 0x0a, 0x0b, 0x73, 0x63, 0x68, 0x65, 0x64, - 0x75, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x42, 0x02, 0x68, 0x00, 0x12, 0x4e, 0x0a, 0x13, 0x6c, 0x61, - 0x73, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x75, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x42, 0x02, 0x68, 0x00, 0x12, 0x4e, 0x0a, 0x13, 0x6c, + 0x61, 0x73, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x5b, 0x0a, 0x0f, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, @@ -978,145 +984,150 @@ var file_temporal_server_api_schedule_v1_message_proto_rawDesc = []byte{ 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x0e, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x73, 0x42, 0x02, 0x68, 0x00, 0x12, 0x5a, - 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, - 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x52, 0x14, 0x6c, 0x61, - 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x42, 0x02, 0x68, 0x00, 0x12, 0x51, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x65, - 0x64, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, - 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x66, 0x61, 0x69, - 0x6c, 0x75, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, - 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x65, 0x64, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, - 0x42, 0x02, 0x68, 0x00, 0x12, 0x29, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x5f, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, - 0x6c, 0x69, 0x63, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x12, 0x25, 0x0a, 0x0c, - 0x6e, 0x65, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0b, 0x6e, 0x65, 0x65, 0x64, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x42, 0x02, 0x68, - 0x00, 0x22, 0xb3, 0x02, 0x0a, 0x11, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, - 0x6c, 0x65, 0x41, 0x72, 0x67, 0x73, 0x12, 0x42, 0x0a, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, - 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, - 0x6c, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x3e, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x42, 0x02, - 0x68, 0x00, 0x12, 0x50, 0x0a, 0x0d, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x70, 0x61, - 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, - 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x63, 0x68, 0x52, - 0x0c, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x63, 0x68, 0x42, 0x02, 0x68, 0x00, - 0x12, 0x48, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, - 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x42, 0x02, 0x68, 0x00, 0x22, 0x82, 0x01, 0x0a, 0x11, 0x46, 0x75, 0x6c, 0x6c, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x08, 0x73, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, - 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, - 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x08, 0x73, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x29, 0x0a, 0x0e, 0x63, 0x6f, - 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0a, 0x11, 0x6f, 0x6e, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x66, 0x69, 0x6c, + 0x6c, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, + 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x66, 0x69, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x52, 0x10, 0x6f, 0x6e, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x42, 0x61, 0x63, 0x6b, 0x66, 0x69, 0x6c, + 0x6c, 0x73, 0x42, 0x02, 0x68, 0x00, 0x12, 0x5a, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x73, 0x52, 0x14, 0x6c, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x42, 0x02, 0x68, 0x00, 0x12, 0x51, 0x0a, 0x11, + 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x65, 0x64, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x65, 0x64, + 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x29, 0x0a, 0x0e, 0x63, 0x6f, + 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x42, 0x02, 0x68, 0x00, 0x22, 0xc1, 0x01, 0x0a, 0x10, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, - 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, - 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x08, 0x73, 0x63, 0x68, 0x65, - 0x64, 0x75, 0x6c, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x3e, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, - 0x42, 0x02, 0x68, 0x00, 0x12, 0x29, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x5f, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, - 0x6c, 0x69, 0x63, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x22, 0xbd, 0x01, 0x0a, - 0x14, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x4b, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x57, - 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x12, 0x37, 0x0a, - 0x16, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x66, 0x69, 0x72, - 0x73, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x49, 0x64, 0x42, - 0x02, 0x68, 0x00, 0x12, 0x1f, 0x0a, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x70, 0x6f, 0x6c, 0x6c, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6c, 0x6f, 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x6c, 0x42, 0x02, - 0x68, 0x00, 0x22, 0xf7, 0x01, 0x0a, 0x15, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6d, 0x70, - 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x65, 0x6e, 0x75, 0x6d, 0x73, 0x2e, 0x76, 0x31, - 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x42, 0x02, - 0x68, 0x00, 0x12, 0x3e, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x73, 0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x42, 0x02, 0x68, 0x00, 0x12, - 0x40, 0x0a, 0x07, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x20, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x66, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x48, - 0x00, 0x52, 0x07, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x42, 0x02, 0x68, 0x00, 0x42, 0x10, 0x0a, - 0x0e, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x22, 0xc7, - 0x01, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x5c, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x42, 0x02, 0x68, 0x00, 0x12, 0x3f, - 0x0a, 0x1a, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x5f, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x73, 0x6c, 0x65, 0x65, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x17, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x52, 0x61, 0x74, 0x65, 0x4c, - 0x69, 0x6d, 0x69, 0x74, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x42, 0x02, 0x68, 0x00, 0x4a, 0x04, 0x08, 0x03, - 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x7a, 0x0a, - 0x15, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x42, 0x02, 0x68, 0x00, 0x12, 0x46, - 0x0a, 0x0f, 0x72, 0x65, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x0d, 0x72, 0x65, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x02, - 0x68, 0x00, 0x22, 0xc3, 0x01, 0x0a, 0x15, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0a, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x42, 0x02, 0x68, 0x00, 0x12, 0x1e, 0x0a, 0x08, 0x69, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x69, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x42, 0x02, 0x68, 0x00, 0x12, 0x4b, 0x0a, 0x09, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x74, - 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, - 0x02, 0x68, 0x00, 0x12, 0x1a, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x22, 0xc6, 0x01, - 0x0a, 0x18, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, - 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x42, 0x02, 0x68, 0x00, 0x12, 0x1e, 0x0a, 0x08, 0x69, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x69, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x42, 0x02, 0x68, 0x00, 0x12, 0x4b, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x74, 0x65, 0x6d, 0x70, + 0x42, 0x02, 0x68, 0x00, 0x12, 0x25, 0x0a, 0x0c, 0x6e, 0x65, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x72, + 0x65, 0x73, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x6e, 0x65, 0x65, 0x64, 0x52, 0x65, + 0x66, 0x72, 0x65, 0x73, 0x68, 0x42, 0x02, 0x68, 0x00, 0x22, 0xb3, 0x02, 0x0a, 0x11, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x41, 0x72, 0x67, 0x73, 0x12, 0x42, 0x0a, + 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, + 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, + 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, + 0x52, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x3e, 0x0a, + 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x74, 0x65, 0x6d, + 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x42, 0x02, 0x68, 0x00, 0x12, 0x50, 0x0a, 0x0d, 0x69, 0x6e, 0x69, + 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x27, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, + 0x6c, 0x65, 0x50, 0x61, 0x74, 0x63, 0x68, 0x52, 0x0c, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x50, + 0x61, 0x74, 0x63, 0x68, 0x42, 0x02, 0x68, 0x00, 0x12, 0x48, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, + 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x42, 0x02, 0x68, 0x00, 0x22, 0x82, 0x01, + 0x0a, 0x11, 0x46, 0x75, 0x6c, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x42, 0x0a, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, + 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x42, 0x02, + 0x68, 0x00, 0x12, 0x29, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x5f, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, + 0x63, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x22, 0xc1, 0x01, 0x0a, 0x10, 0x44, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x42, 0x0a, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, + 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, + 0x6c, 0x65, 0x52, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, + 0x3e, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x74, + 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, + 0x75, 0x6c, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x42, 0x02, 0x68, 0x00, 0x12, 0x29, 0x0a, 0x0e, 0x63, + 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x42, 0x02, 0x68, 0x00, 0x22, 0xbd, 0x01, 0x0a, 0x14, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, + 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4b, 0x0a, 0x09, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, + 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x6d, + 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x42, 0x02, 0x68, 0x00, 0x12, 0x37, 0x0a, 0x16, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x13, 0x66, 0x69, 0x72, 0x73, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x49, 0x64, 0x42, 0x02, 0x68, 0x00, 0x12, 0x1f, 0x0a, 0x09, 0x6c, 0x6f, + 0x6e, 0x67, 0x5f, 0x70, 0x6f, 0x6c, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6c, 0x6f, + 0x6e, 0x67, 0x50, 0x6f, 0x6c, 0x6c, 0x42, 0x02, 0x68, 0x00, 0x22, 0xf7, 0x01, 0x0a, 0x15, 0x57, 0x61, + 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x65, 0x6e, 0x75, 0x6d, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x42, 0x02, 0x68, 0x00, 0x12, 0x3e, 0x0a, 0x06, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x02, 0x68, 0x00, - 0x12, 0x1a, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x22, 0xda, 0x01, 0x0a, 0x0d, - 0x4e, 0x65, 0x78, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1c, 0x0a, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x12, 0x3d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x42, 0x02, 0x68, 0x00, 0x12, 0x21, 0x0a, 0x0a, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x42, 0x02, 0x68, 0x00, 0x12, 0x27, 0x0a, 0x0d, 0x6e, 0x6f, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0c, 0x6e, 0x6f, 0x6d, 0x69, - 0x6e, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x42, 0x02, 0x68, 0x00, 0x12, 0x20, 0x0a, 0x09, 0x63, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x42, 0x02, 0x68, 0x00, 0x42, 0x30, 0x5a, 0x2e, 0x67, - 0x6f, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, - 0x2f, 0x76, 0x31, 0x3b, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x31, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x42, 0x02, 0x68, 0x00, 0x12, 0x40, 0x0a, 0x07, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, + 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x07, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, + 0x65, 0x42, 0x02, 0x68, 0x00, 0x42, 0x10, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x5f, 0x66, + 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x22, 0xc7, 0x01, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x5c, 0x0a, + 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, + 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x77, 0x6f, 0x72, 0x6b, + 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x42, 0x02, 0x68, 0x00, 0x12, 0x3f, 0x0a, 0x1a, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x74, 0x65, 0x64, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x73, 0x6c, + 0x65, 0x65, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x74, 0x65, 0x64, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x53, 0x6c, 0x65, 0x65, 0x70, + 0x42, 0x02, 0x68, 0x00, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, + 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x7a, 0x0a, 0x15, 0x53, 0x74, 0x61, 0x72, 0x74, 0x57, 0x6f, 0x72, + 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x06, + 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, + 0x49, 0x64, 0x42, 0x02, 0x68, 0x00, 0x12, 0x46, 0x0a, 0x0f, 0x72, 0x65, 0x61, 0x6c, 0x5f, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0d, 0x72, 0x65, 0x61, 0x6c, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x02, 0x68, 0x00, 0x22, 0xc3, 0x01, 0x0a, 0x15, 0x43, 0x61, + 0x6e, 0x63, 0x65, 0x6c, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, + 0x42, 0x02, 0x68, 0x00, 0x12, 0x1e, 0x0a, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x42, 0x02, + 0x68, 0x00, 0x12, 0x4b, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, + 0x66, 0x6c, 0x6f, 0x77, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x12, 0x1a, 0x0a, 0x06, 0x72, 0x65, + 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, + 0x6e, 0x42, 0x02, 0x68, 0x00, 0x22, 0xc6, 0x01, 0x0a, 0x18, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, + 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x21, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x42, 0x02, 0x68, + 0x00, 0x12, 0x1e, 0x0a, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x42, 0x02, 0x68, 0x00, + 0x12, 0x4b, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x12, 0x1a, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, + 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x42, + 0x02, 0x68, 0x00, 0x22, 0xda, 0x01, 0x0a, 0x0d, 0x4e, 0x65, 0x78, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x12, 0x1c, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x02, 0x68, 0x00, + 0x12, 0x3d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x21, 0x0a, 0x0a, 0x6e, + 0x65, 0x78, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, + 0x09, 0x6e, 0x65, 0x78, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x42, 0x02, 0x68, 0x00, 0x12, 0x27, 0x0a, + 0x0d, 0x6e, 0x6f, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x03, 0x52, 0x0c, 0x6e, 0x6f, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x42, 0x02, 0x68, 0x00, 0x12, 0x20, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x42, 0x02, 0x68, 0x00, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x6f, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, + 0x61, 0x6c, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, + 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x73, 0x63, 0x68, 0x65, 0x64, + 0x75, 0x6c, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1147,14 +1158,15 @@ var file_temporal_server_api_schedule_v1_message_proto_goTypes = []interface{}{ (*NextTimeCache)(nil), // 11: temporal.server.api.schedule.v1.NextTimeCache (*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp (v1.ScheduleOverlapPolicy)(0), // 13: temporal.api.enums.v1.ScheduleOverlapPolicy - (*v11.Payloads)(nil), // 14: temporal.api.common.v1.Payloads - (*v12.Failure)(nil), // 15: temporal.api.failure.v1.Failure - (*v13.Schedule)(nil), // 16: temporal.api.schedule.v1.Schedule - (*v13.ScheduleInfo)(nil), // 17: temporal.api.schedule.v1.ScheduleInfo - (*v13.SchedulePatch)(nil), // 18: temporal.api.schedule.v1.SchedulePatch - (*v11.WorkflowExecution)(nil), // 19: temporal.api.common.v1.WorkflowExecution - (v1.WorkflowExecutionStatus)(0), // 20: temporal.api.enums.v1.WorkflowExecutionStatus - (*v14.StartWorkflowExecutionRequest)(nil), // 21: temporal.api.workflowservice.v1.StartWorkflowExecutionRequest + (*v11.BackfillRequest)(nil), // 14: temporal.api.schedule.v1.BackfillRequest + (*v12.Payloads)(nil), // 15: temporal.api.common.v1.Payloads + (*v13.Failure)(nil), // 16: temporal.api.failure.v1.Failure + (*v11.Schedule)(nil), // 17: temporal.api.schedule.v1.Schedule + (*v11.ScheduleInfo)(nil), // 18: temporal.api.schedule.v1.ScheduleInfo + (*v11.SchedulePatch)(nil), // 19: temporal.api.schedule.v1.SchedulePatch + (*v12.WorkflowExecution)(nil), // 20: temporal.api.common.v1.WorkflowExecution + (v1.WorkflowExecutionStatus)(0), // 21: temporal.api.enums.v1.WorkflowExecutionStatus + (*v14.StartWorkflowExecutionRequest)(nil), // 22: temporal.api.workflowservice.v1.StartWorkflowExecutionRequest } var file_temporal_server_api_schedule_v1_message_proto_depIdxs = []int32{ 12, // 0: temporal.server.api.schedule.v1.BufferedStart.nominal_time:type_name -> google.protobuf.Timestamp @@ -1162,29 +1174,30 @@ var file_temporal_server_api_schedule_v1_message_proto_depIdxs = []int32{ 13, // 2: temporal.server.api.schedule.v1.BufferedStart.overlap_policy:type_name -> temporal.api.enums.v1.ScheduleOverlapPolicy 12, // 3: temporal.server.api.schedule.v1.InternalState.last_processed_time:type_name -> google.protobuf.Timestamp 0, // 4: temporal.server.api.schedule.v1.InternalState.buffered_starts:type_name -> temporal.server.api.schedule.v1.BufferedStart - 14, // 5: temporal.server.api.schedule.v1.InternalState.last_completion_result:type_name -> temporal.api.common.v1.Payloads - 15, // 6: temporal.server.api.schedule.v1.InternalState.continued_failure:type_name -> temporal.api.failure.v1.Failure - 16, // 7: temporal.server.api.schedule.v1.StartScheduleArgs.schedule:type_name -> temporal.api.schedule.v1.Schedule - 17, // 8: temporal.server.api.schedule.v1.StartScheduleArgs.info:type_name -> temporal.api.schedule.v1.ScheduleInfo - 18, // 9: temporal.server.api.schedule.v1.StartScheduleArgs.initial_patch:type_name -> temporal.api.schedule.v1.SchedulePatch - 1, // 10: temporal.server.api.schedule.v1.StartScheduleArgs.state:type_name -> temporal.server.api.schedule.v1.InternalState - 16, // 11: temporal.server.api.schedule.v1.FullUpdateRequest.schedule:type_name -> temporal.api.schedule.v1.Schedule - 16, // 12: temporal.server.api.schedule.v1.DescribeResponse.schedule:type_name -> temporal.api.schedule.v1.Schedule - 17, // 13: temporal.server.api.schedule.v1.DescribeResponse.info:type_name -> temporal.api.schedule.v1.ScheduleInfo - 19, // 14: temporal.server.api.schedule.v1.WatchWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 20, // 15: temporal.server.api.schedule.v1.WatchWorkflowResponse.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus - 14, // 16: temporal.server.api.schedule.v1.WatchWorkflowResponse.result:type_name -> temporal.api.common.v1.Payloads - 15, // 17: temporal.server.api.schedule.v1.WatchWorkflowResponse.failure:type_name -> temporal.api.failure.v1.Failure - 21, // 18: temporal.server.api.schedule.v1.StartWorkflowRequest.request:type_name -> temporal.api.workflowservice.v1.StartWorkflowExecutionRequest - 12, // 19: temporal.server.api.schedule.v1.StartWorkflowResponse.real_start_time:type_name -> google.protobuf.Timestamp - 19, // 20: temporal.server.api.schedule.v1.CancelWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 19, // 21: temporal.server.api.schedule.v1.TerminateWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 12, // 22: temporal.server.api.schedule.v1.NextTimeCache.start_time:type_name -> google.protobuf.Timestamp - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 14, // 5: temporal.server.api.schedule.v1.InternalState.ongoing_backfills:type_name -> temporal.api.schedule.v1.BackfillRequest + 15, // 6: temporal.server.api.schedule.v1.InternalState.last_completion_result:type_name -> temporal.api.common.v1.Payloads + 16, // 7: temporal.server.api.schedule.v1.InternalState.continued_failure:type_name -> temporal.api.failure.v1.Failure + 17, // 8: temporal.server.api.schedule.v1.StartScheduleArgs.schedule:type_name -> temporal.api.schedule.v1.Schedule + 18, // 9: temporal.server.api.schedule.v1.StartScheduleArgs.info:type_name -> temporal.api.schedule.v1.ScheduleInfo + 19, // 10: temporal.server.api.schedule.v1.StartScheduleArgs.initial_patch:type_name -> temporal.api.schedule.v1.SchedulePatch + 1, // 11: temporal.server.api.schedule.v1.StartScheduleArgs.state:type_name -> temporal.server.api.schedule.v1.InternalState + 17, // 12: temporal.server.api.schedule.v1.FullUpdateRequest.schedule:type_name -> temporal.api.schedule.v1.Schedule + 17, // 13: temporal.server.api.schedule.v1.DescribeResponse.schedule:type_name -> temporal.api.schedule.v1.Schedule + 18, // 14: temporal.server.api.schedule.v1.DescribeResponse.info:type_name -> temporal.api.schedule.v1.ScheduleInfo + 20, // 15: temporal.server.api.schedule.v1.WatchWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 21, // 16: temporal.server.api.schedule.v1.WatchWorkflowResponse.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus + 15, // 17: temporal.server.api.schedule.v1.WatchWorkflowResponse.result:type_name -> temporal.api.common.v1.Payloads + 16, // 18: temporal.server.api.schedule.v1.WatchWorkflowResponse.failure:type_name -> temporal.api.failure.v1.Failure + 22, // 19: temporal.server.api.schedule.v1.StartWorkflowRequest.request:type_name -> temporal.api.workflowservice.v1.StartWorkflowExecutionRequest + 12, // 20: temporal.server.api.schedule.v1.StartWorkflowResponse.real_start_time:type_name -> google.protobuf.Timestamp + 20, // 21: temporal.server.api.schedule.v1.CancelWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 20, // 22: temporal.server.api.schedule.v1.TerminateWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 12, // 23: temporal.server.api.schedule.v1.NextTimeCache.start_time:type_name -> google.protobuf.Timestamp + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_temporal_server_api_schedule_v1_message_proto_init() } diff --git a/proto/internal/temporal/server/api/schedule/v1/message.proto b/proto/internal/temporal/server/api/schedule/v1/message.proto index c8febceefeb..4fc55469cd5 100644 --- a/proto/internal/temporal/server/api/schedule/v1/message.proto +++ b/proto/internal/temporal/server/api/schedule/v1/message.proto @@ -48,12 +48,11 @@ message BufferedStart { message InternalState { string namespace = 1; string namespace_id = 2; - // this is just the workflow id of the scheduler, but accessing that from the workflow is - // currently not available during replay, so we just pass it in here. string schedule_id = 8; google.protobuf.Timestamp last_processed_time = 3; repeated BufferedStart buffered_starts = 4; + repeated temporal.api.schedule.v1.BackfillRequest ongoing_backfills = 10; // last completion/failure temporal.api.common.v1.Payloads last_completion_result = 5; diff --git a/service/worker/scheduler/testdata/replay_with_incremental_backfill.json.gz b/service/worker/scheduler/testdata/replay_with_incremental_backfill.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a1aced18b9cbe2d413287698e9f62de8bea03888 GIT binary patch literal 12234 zcmV;*FE!8~iwFP!000021MFREbK)4kTRcK=!;lC#lM00Ch|1Zgs04wZ$L5eERg@KD*O6dieV3|H!LP zpZ?hVyMJGF$OjMb;n$;gRWEe2_Ypw21Yika70_QH_=?%jkcSx15F+CxQ8M2UX|u&- z7|d3+wQDb9Z#M15<|1~TAI)Pknru79kjE6Hetfh#LtM*xD97=_vzD=&&1OH31ESps zetbeco0{9H#BMmBEzF=4B~dA|$F8#s+(l_m`tTRcpTGWKoDEc2y32fA=Y-!hD#hPV zGxMhNW&D!S+)RR*>HJX7{^yUQSEOa=#-?M&<_ivc4NUEs<4s4|p!6_U>VqcTqAH0+ zb)NNUKYaN0HBIP7kvVerdUG26*lzHNuC10f>L*UvPg-GpWp?RQN8`YF>_auS(`V&W62g;y1wO7CW_VA*YKVNqHu$#g|)t@~H6|qqriDGpumZj`(MVwSU z@u|{)w1f0+Wi+cTQ|m9V8sWCl_!3zbQ`3&*mmz?M2fdfLO1FCoL9tPFotQAWmgrb6w- zO#VE0_3x9~{mTZ4$z;oCw$b$i1kr)$A7%@0O<27mjaDMF7*-dGu8i8#CLGlgq3l1$+*4c` z3+09WWR_Jz_bu7dNrLqKn_kW6yq3SEdz!WW#&VnN&8hrsdHl+1)aTZSxQG*;DBiiCF6xto&nHH?h$|6; zPGAix&vsakdPtrbS_fK#3TDU?g}SN1dro)LVG?M!r}oR>EA!uLn+;(51^aTmX{Ea{uu`~Tk^cW5qy z_>gwteqXxLjw{PjmWvph$N>*an3!FOC;yg)H0`*Ebvbj$9TJbdD4s2nu9v!%WIOH$ zA~GiTQox1u!2!TSiq4~t>MWcG?nRBzr#12yv_~zyFHF(+*{5^@XlA zBV!R5lOHu_5) z@e@;{r?t`iyN&qee*0OAIKfStHA6GTBzpo1ssoaDK)deDt+4a#BqAfFHAu3Sl_Zo8 zeNE1NrHl3^OVO{ zCW?+-CksK?sQIF8oBL~%K8`M`enXgeN zeQEuaWQVa`6&dHT361W`>rDF~qg6B+m>R!S5D%@XvKjMcoSK}Cv!YX*u?iN|Upj7y6+eIpy|Q6HU#MKvXJAOkXYNcw6L z%_n4TSwE=~_CxY{K-w+|@1>*lD>k5mjr^;GCt3$aCgVmB8U4!G8OVWgFfUcBV}+QU zK{DkY3K*li8S^TmRl>S&B4chf)@hIQ*(LHM8`RC%$XM2!HrhI}D91Dg&F5`jWv@N$ zB#uU88#IHgzkVZ#je+>%{RthokDZM^iQQxwxF!Yi+zoq~JOP=bM);+>e&A3!v6Vi0 z{qJASSoTA>by%f{c&w7UaakqX3`m|iTTY2D|K~U+=>{nr&fatWsm@p)%3ZOD46dRa z#~Y{p{w{A6XNpE~7Gmz!D9+v?ivPsno5Z9e-$8uI7U?5CPxQr=c^o26a z;y~N)tv0k9|E64Ow{H0omV%i#?5^|hUbp>n9^P3y)bbUmc}>3c7

Q$0WXvt+Vs5 zk>83bdL$Nvee7IpS6fP{m=@yg=j81xOF11C8+V$Mcd9RC)y745%R)ZhvOswI3SEeI zPI;3tFWah4<>yUecdl_~?T9W_2^8Y3=H#vFnJ(9bc>6hd`(&*O6S@%ZG$-#=B)oN4 zgtwei-kF_9>ZFiN=+8r4 zTc^eV{Lk%73hu_?zUSyH)AoQ$I z?tfgchmyaU%gA;9zOw=)iQ?G;s|pp-2m+X5A?lCGyD_4Qg+G10Y7D61nNcfm>)9KWS@JVO}8y5C#3y8ugJe=PaZnxt4M z;h)jqcB8g7gk~st0PCre^^;1WNcxc|sR~lvmnfs5(6Z1B47HrmAX%$HLy*VqF7(=s z3=Zl9hM;6)!@36u@A^dYb)c((FA1ZJCe=|dk%du8#*xZMNeRt_ycf6C9wam*4HPL! zb;g88YWNdE(+p&!LmiRvRRJw&dN#JTt}3w}>R3(yGD!NVhn-Lo6sY@~euzx!o}?*B zHeX$8M?W z6DgH|l#XaUEw9?Do%KUxrl7VU5?Mnc7aI{8LOUVzsUeB<(-uZ+9reAo8cd|dMn6hEbQWhh_CW$Y#R8RG;Q1-2D z9ci`orTyixto;A-F?2B>!$69~;My+y+hRr%z1VXjiRgOhUGg{R^;Y~f`Aq;CJgWKH zB6Y4K{Y*v8-h4kkb*8)ZSW3#~(Rv z;T(6M4?4bipB%Srzk7a4+Q4s zrEV(9%-D_@oUeT!ixXXwXj!d~Wr2(_>;$q0fUJqQr8PrY)nmga3lS-lq4wykwmHMs z*xmeBPpBGhFEZfv7a62aKHA`GqK9`EAk5Rbw7JM&ve;f^uxfj3tZZ;OXN$`T;88L49DO9 zhKDPt|AGB|z;Jt+@C7Y^!Tfm($eCvI%txC2hBRBQz6%Vumby5VQX5IBNJ;5OsmXsC zhO51Qikv?}P@tH9;5ZVp&-#b`a?Q8z;t%!<<|)dH-eOPjY95E&E?hyWnXx_A-sQr{ z!Z&V>UDMfJ;PA6wR(WbEt)1+el0r|9HOA3XL)%SLf>mwp=bC#)Y5d4h|3ch^j1L+z z#mYJmcf7%j7s$ed0}Vi~#Fh|Sou0{6%EDPAny+ztsAa!5qT#2@i@#O!$P{V_Z41GD?z z!0g^q6PM9VtsQ3f?O#%=NJ(|&jrVlui}>ZPcpZfbE;){&dG;o}j=%A#SqkN-o1;GC ztk{P|#T{&J(NTp&M-op&fTW=M`1$00|2x)Mk(&SJFS3wSZ_^hLOit?f!P9{q~(Z23%igY2U zysA>jvH!Yi&L^$|z3o2G+_p&e2&iyz9Ni7Vla%?UsfRnI#Nps$@Y2ms~J(O_LrD8OFK>YjX^fzq*tva?n6iS#>FZ z_kp!Bx_fo$u~tsFD&_;$eq{k%K?`6I1NhruZHzvuOWy=*ci*LYEQ7V7Nbv zj6;$v({0TNBtIBSjixUP8!%x1O&pKde&+s;>M;^Ja{Cy*ppW4pMZxe|zZTKHSR`ufd|OeeXs7hd?nbHgin2)%ZR&SeW8J zlGS@8s~8o~AqOC!d4^@~k*s3szR4V8qGKPhySp5 zZQW_>-1fhm*A7zMEq!C`2Z{-GitLgAuKfn)W@DP}1Pb`q9|=fz5=gLu+3jxqkkN+N zvM!71wdPuyI?q?c!hx*Bho4Sng^66sBvmmjOai|}o9*kwQ?a$zzMh0P04Wep z+)SUc5)he9Z~obhRciqD&aN*1`lT?28Dfl9V<@n!icst9WER0CnN^ZmXD736wrlXL zv}+##;ITE>9w}5e2b;pm$n-V$I}&c${2Pd>Q!trknZXW4kezlQ1EKu1(lTP@C@q5* zmIVN3lnJvV84M6RcZw6U3(Bdb73t9lJmh07D{~(5z_dLbdH%!;5r!jY1aVRf56O7R zLl(h9Qed)%La1eg2o%9XTBu+iQk;c{>|sVm0;3Ax(V(LKYgI&#!Afrwkzr5t`-~lx zWP_06{0HAtXs{#ONKs0#g z*$kSwF+s`=rJmh^Zn+us_a`E$!5adb;)6x`sfh|+CMw>959g_>+u=iysSuTTc39%W z&m|&TRL(A^o_`$&F-*f*3vSPm2{vcx1Z$|zegSSfoCdS2#Pd5h|8N8L<+mnEs{g{- zCngv}3^B$+02EnO_gM1iOhhDwB@tN?k$(UYxm|;2nsfC>m3^z;jksN2ws-gI{Ke*?2VeD@79RP(|ERi0 z@a4m|=2tiAHM-M*ze<~5gH<#A=l4$I5(H>$u;a5naXx(*Ckwvt+8-uPqvpKN2$hXS zH89=@`k3bVL4!tV{J+81y9bIVzuj4;b|)+g7DGb|RMOCe(9qmy4Yjx)f}{udc}YV*w*UqVEr2UvIMabZ$9Q`n|8>P31JiUhHVEf33WJ zsD5bI)*ipJ-gy0+f?JvnKek@G4m6WG_PV$6=w_9(?N-jVTdQUwH#MMA)my6Zus?MydpH91tbiPT6 zB`=VkS+tNz2Ya?$DR*YI{d!-54d0=^;z?coel-cu{ChYH)bAfendmr-r12V-2 z29P)vA8?5eZ^DQAOZc#ti#5lGVC`lRkoFXn`0#U6;Xr2HBuoY~_XZHxPUB{~mAUQC z+_S5zAcnDQgww$9DubA=gGAocmhsiD+?R(pYyXCTl?lcWQ;Z=TkyA0IEa!V0#>jJG43YL2s5lCj81r-Y6;I_x1TR_! zYdCS+W2!I)xxTKjsRFkKXELcT-K2%2?;x+yd5kgli(akqt6>Z^#TY>aIu&DBnaepX zm-8|iB5j8#8w5FXItMDb4)Uaju0zcv0IzV~2kzY3 zQh$x}K5$PxXWMl(Ckm{)CT%-8+ji8PRH=+T_v7T+fL55H+dIe@ZHIB+DW#xlK@Dg zwp%&dZoy5S%V;e-Ios~&95M9DoX(T?j9dM+d=X~oiy(xo0*`yA`1~3tRQxGh3;yJ? z_D?DJR9hx&TN7(nzx80ZKE8M2ou0?tV1!2L^YI@j(8=`pjfvCU_&;vyaMk$N-|6jU zYsJI!*8)f5@!r;o!$ml1{Iac9Ad{;gtIyq;1LPV27hO%8zH+46YEBvs4zkHiMWC6R zOv!d@MeOo-+woiP7NWY+$J@`gYvQfxtrP5Rt&lNTsxdCTZfm2eC6)8L-CEIF>8%rP z_90Qvgq}T}yLJFozp1oapy>x|6&)k>*(yi^61EG|Q_PIy5l>re*E>g*z_vzahiLkn>L5iR1t zc*c?D*_A#5$Vy053Hlr*&G?D6Swo&`Wd3$O%Z^h8F_vdXN%o3aM2vs6YYKWL#dlIs6U zSNXAJ*5Ef62~n`08y}D%K5!1$sf@qHOMG|}K6K8KRovkNRWT%yNmOPPe|qIYrbR%e z>re&ruI(l&z^6gGJ`dEtUJX#M5yk=R0%PmQb^5(!PwFzOxKx*OSn9R$lummh0{K$D_~5bg4yB zKE0o9L{grW4Rwz*d9J+~ziElmj>*GYCQsVmnFZ{}OrF)8jVHO7yv~Nn3++oa#}1Ya zsh-&(-VFe5(pCMQF12t&s^tk>ibyY{&q-^!?#LnH-k*PxE2 z-GXXA=_z<0k1l{5TcC{X8QeT%O=+7lVv5PL# z&<9*M|OK7B()^K-bI9oNPzFeb|=K)vEV*@3soX56_q12iS4)l+itq zZz#xQCVjbTw$&HS*5unD4r_3t`mt{M*RhKv31{Ir80k_?HNU}J6R-+cfalj&O}`Gj_E_~} zG7Dzz#B0m7Yt=&V=(kXyY{0e0KtDh9T}?h` z?o7^Qc0CF0ItXUY%54wgg2>KwZJBrvV?ZSB7fOC|WC(W(pfEXw{A56!Mt*X1oczQW z5Hmu6#km>xY4D=RohhT@0z=LrC!{!WWCn35j(kROn^gjSEAi+X8tuHGU9#X`BESOCM3Q!HRHTTK}* ztE>u%fdw>KLuzq72dsz~7#Cb+CBRu?U?KX}a2O3M#IjI@^eCxt^?}}bIvA<|nE~8C z8UrJ4P_`+M$&xTeI2Mc-69Z#jbPW7-S@!Wpz%mBjvjA*n0TclUP88W@Sd=mFn_}SU z8De0EsoxjMzawtep;~*cjDdgi7+4rMUc?X|bhb?tSOr1A*{dJ~DytyMDu}WQ;@njb z$Ay`66~yx4nosTbAN85gl0D2>KknYI?P}sb2tKWNk5>0iH9D~pLQq49#Q!#m4u-sFJF0hJJ0N|JuR|-LBSqkwxmqG~3)V>IXq8J*8 zu#yHUga+a!Ybcn7D7K0?51LU>Fb#B`^PqtR`_>R*eI7Kb5CTX=^afT%#og9$5D^rO z*ijmYV3xmvO$dV}ApkK785hOSK$H|t106XJT9oHO?^yr@z|aJ^rUFmofoOS9$(wkf zqeLM&lYCi&wp%$rKA62 zQmCO32n4Mnx3#kKpV0Bx(tp>C&gAx^@(8j)BVB~+>WA{Uil2O3#mzRu^pMS2da*Tc zIr}P~IfNE}2o0JkivIm&e7h&Kr)}Wtu1`2>e%sG*9jb}nJl}LP|5}Z8`H;<7+_gbf zXK$K|hdr!f7IVFawfb<)x6gd2Gd|hFoVklDU+;}|d%I&_?2oIvsa#=)h3Dr!u#jl+bfD43|GU!0bKjGA_5hG`>B)nenboQjBa_lTPYR`v4g40qq`K?nJS|=d@6r&2ttdhhi zhQxrtXb~x+98q2biNOmeF~nJbo|Dl~>wr`uidcp4A*l36j8|}9P>YX-V8Fpq5`$5a ze@`BS&}30CA(pbtf=bo}E}p~?M-U5wlEm0I0%T?cR5`-%RPDdVrLg6Z=ld^le|F_}xT@+Oi#Fp5L9Q5PGdsh|QX_4(*N#Ydgrjfs1uNVsV(QhOojC|)sB6c%7zN9+v&vJ2%0)s=sP5|gS8t2 z5ZVz?&w;8R>N!`b3c-nMhq}+JAaSVY*Y!OsU6dO7DwdH}{}m=B8nCnR;pt^z&5Trk<_GQ%5TNSMxfERhw$=+=&{a9*_FFJCnB5 zR0`2L|C@6N^c=J!{aGfntC?%Nsd_F76MYJe1JtuuGU=coh7;*u>3d)}YpLgSklrF1 z+Es0s&Bzch5W>8%cbs{+i=vHs<}|>6U@|KvnrMTrS4<=rS;ce zf7~(5t@9t=qF;Ua5fYmsBx5hj(&t6UvYz8@^&IsxsH5t74!MxFL)|Qb3N*@kj-RiN zdYuG*J=5fEn560E?bse~2wiyt(_nU)x_*avet_L%s?|}G9a$Qt-^u4+3t@;M!eGit zLFyQS6{n8zcp#-bMk$Z6k;m}5TX~GI;VS=5l4d)dd46Z^Ho}Ql+njT>YWm$c@H@(< z9IJ10guI(yULNCd^!yv@V?am^F^LICicD^ce<8X3bPfNED<1*Zk1C#}ahxgye&&s?*@XzRYZ9`=S&1@So@*^SXa^OM zC^MB(oji52MEeL1*a(jXL%4VIApcP{p`sueV2Io4Px8mzr=mak2lXePCGlggqTq~4N49RK`l!SN@)^AiY1eRLkkT`4lOHcJ zIj1y=*eD#ScYpb`0k^e5DFHAzD1(4WP?!n|;{aG`P;z8_a^bVUY^zwZXi&aP0%(*3 z1OUvzo()Qlis6OtgWuf)!SC^%`akj082hU$RKV|_48Ql%DDAbc7R^gI^HYDO;deg` ze+4rS^)6wef`)_D$;9uv8qm}5`#js;ozK3L=-}`>GC0hGkYR;E0PZFy<%haB`T(d4 zfiDVm73UZ}?;JyB|0q*NJC*l0i?X&q&*pQX-r%ST^Vpwe_Kn(p8~f)O)c2hQ(|?1t z{ht}Mw7f}=PIq3vegK`V?Hks;|C4hZC^y3avSTiRpsE1M_lse3O!pDP-XhWU^ejF# zsQtw7(wQLG&O|7V3mGIkn|DMFf+zsuf?;T7^G+=5AU)1Uo11qgklGlbuoQ-pl+<=l zl={O^{V7NkhMY%3wo?cR0|L0goyClT!lc@NQfO!MPOK{-Jqyg1%{wZ_LTyQc+|WJ3 z#}sgaAl?nD<(4|(+mz-CsS|5bC+;63b%KOxqNzuK9XUfzc<~&!^Bk8ZL4-^QDWwfP zLr$UX=lEej_z~0Z-f*p8C`u$8@sdD%P{I(6LMFqIKAl6p{W&g7-jFdc2?|mrpJHT3 z>V#AKAUxUy!lMF&-;xB-ED12yQ)etfWRx8qBEw${A}g3!U!94S*-x_TKboEO<;H%J z#pGx3Z47tWUp7-`xwel2eQpYR)yXe?lBuW9dZhtwbk9p~S6 znH4|+34Wap`#z}?{#dy2kWwd43*ncOI_b?$ZeO;=_Rb(|m_bT!Htu0tA~)>_V`ekt;nPIdiq)ORXwt?ty~w;vWC0bXEgmfZpX zOpzV%g5d(ZumWCSW)wupm=soKa&pEkgBO?{1ia{n{l36Jl;CI(X$B!L4G`~_1Re@1 zBg!y}c7hiO8O*wc*a&~-1VZ}8*=;lMfVv5e4K;@c=6jQ!Y~PlfMU;q44k7v z;$#PXu-m&nTlKs0&Q0e~?YTD`waObbz|_0*+X3~{fND99H4SijHVtv-Hahi^=JXeL zC-m<@SH0qN7fCa5U)ip@a6=)a)1jshzPav9HcoGo_Dufjcj3rpP=9*U7>|Kw+u63i!`j%kP7$oOy6%kOv;~s(JDtTbI`dtN9{V9%Sos`e3F0<;jC=oq-S9 zoqRrMcT9Y+%#{b(y8J%KRS#BX^}x$n53+SXV5F|As(hd-Y6ocfpXRJL9-R8q(N~lK zia$UG2pjNOurz#&(6cnC#TE$1AJ78f|A-by;;_?rUaM7agYt!2QJ;Nh>|DEX%XKcC z(fQ`nYsPM?{0J>57{di)__GZk!7;fka7-&WCc!2_R5!nXk^vhyrofhQOmdicQ(TThF_K|P!agjK zzRm%m8P$3s&pGllhEE_2Fs>bT8Q-CuTF@1t;fYl=uaIi8y+5`P^CYAgz1dkBv?nVst-;{( zW~fonAKARi#rGp`)$;UxG0c8bw78p;bJ*XYu+8M!Q@`!dMmIp7TJP5b1Hz>2&5VZ-Po3xOW@> zzo@NMz8vWM8MV7%TR(=nc&8oJ0GF}hNNXsf0E>ab5l^eappPh_-1H@B*4>ta> zOw%R8sK7yh5O|$2lneco`_goY*o}TNKm-8C01$nQTNrY6jUEyC8)f_$0;ajcucLsw zAy5I}UJ&ROlJhhC7w5lt3JfnE352i}2vh)6K%iD2Pzp_g2(h}IB*PXUPyuXHIVm`d zo&=$II1(!8Vi1-H=TS*UeO?j+3OS$vJd;D%3V||YbY~%>pa@9?A$G(d1+-7)q`Mj~ Y6n<3*^e?@u`Ojbf1p}I3$zh}c0PPs2h5!Hn literal 0 HcmV?d00001 diff --git a/service/worker/scheduler/workflow.go b/service/worker/scheduler/workflow.go index 3c485fe489c..5c40f55b862 100644 --- a/service/worker/scheduler/workflow.go +++ b/service/worker/scheduler/workflow.go @@ -72,6 +72,8 @@ const ( DontTrackOverlapping = 3 // start time in backfill is inclusive rather than exclusive InclusiveBackfillStartTime = 4 + // do backfill incrementally + IncrementalBackfill = 5 ) const ( @@ -139,24 +141,23 @@ type ( } tweakablePolicies struct { - DefaultCatchupWindow time.Duration // Default for catchup window - MinCatchupWindow time.Duration // Minimum for catchup window - RetentionTime time.Duration // How long to keep schedules after they're done - CanceledTerminatedCountAsFailures bool // Whether cancelled+terminated count for pause-on-failure - AlwaysAppendTimestamp bool // Whether to append timestamp for non-overlapping workflows too - FutureActionCount int // The number of future action times to include in Describe. - RecentActionCount int // The number of recent actual action results to include in Describe. - FutureActionCountForList int // The number of future action times to include in List (search attr). - RecentActionCountForList int // The number of recent actual action results to include in List (search attr). - IterationsBeforeContinueAsNew int // Number of iterations per run, or 0 to use server-suggested - SleepWhilePaused bool // If true, don't set timers while paused/out of actions - // MaxBufferSize limits the number of buffered starts. This also limits the number of - // workflows that can be backfilled at once (since they all have to fit in the buffer). - MaxBufferSize int - AllowZeroSleep bool // Whether to allow a zero-length timer. Used for workflow compatibility. - ReuseTimer bool // Whether to reuse timer. Used for workflow compatibility. - NextTimeCacheV2Size int // Size of next time cache (v2) - Version SchedulerWorkflowVersion // Used to keep track of schedules version to release new features and for backward compatibility + DefaultCatchupWindow time.Duration // Default for catchup window + MinCatchupWindow time.Duration // Minimum for catchup window + RetentionTime time.Duration // How long to keep schedules after they're done + CanceledTerminatedCountAsFailures bool // Whether cancelled+terminated count for pause-on-failure + AlwaysAppendTimestamp bool // Whether to append timestamp for non-overlapping workflows too + FutureActionCount int // The number of future action times to include in Describe. + RecentActionCount int // The number of recent actual action results to include in Describe. + FutureActionCountForList int // The number of future action times to include in List (search attr). + RecentActionCountForList int // The number of recent actual action results to include in List (search attr). + IterationsBeforeContinueAsNew int // Number of iterations per run, or 0 to use server-suggested + SleepWhilePaused bool // If true, don't set timers while paused/out of actions + MaxBufferSize int // MaxBufferSize limits the number of buffered starts and backfills + BackfillsPerIteration int // How many backfilled actions to take per iteration (implies rate limit since min sleep is 1s) + AllowZeroSleep bool // Whether to allow a zero-length timer. Used for workflow compatibility. + ReuseTimer bool // Whether to reuse timer. Used for workflow compatibility. + NextTimeCacheV2Size int // Size of next time cache (v2) + Version SchedulerWorkflowVersion // Used to keep track of schedules version to release new features and for backward compatibility // version 0 corresponds to the schedule version that comes before introducing the Version parameter // When introducing a new field with new workflow logic, consider generating a new @@ -201,10 +202,11 @@ var ( IterationsBeforeContinueAsNew: 0, SleepWhilePaused: true, MaxBufferSize: 1000, + BackfillsPerIteration: 10, AllowZeroSleep: true, ReuseTimer: true, NextTimeCacheV2Size: 14, // see note below - Version: DontTrackOverlapping, // TODO: upgrade to InclusiveBackfillStartTime + Version: DontTrackOverlapping, // TODO: upgrade to IncrementalBackfill } // Note on NextTimeCacheV2Size: This value must be > FutureActionCountForList. Each @@ -282,14 +284,17 @@ func (s *scheduler) run() error { // resolve this to the schedule's policy as late as possible enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, false, + nil, ) s.State.LastProcessedTime = timestamppb.New(t2) // handle signals after processing time range that just elapsed scheduleChanged := s.processSignals() if scheduleChanged { // need to calculate sleep again - nextWakeup = s.processTimeRange(t2, t2, enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, false) + nextWakeup = s.processTimeRange(t2, t2, enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, false, nil) } + // process backfills if we have any too + s.processBackfills() // try starting workflows in the buffer //nolint:revive for s.processBuffer() { @@ -392,21 +397,32 @@ func (s *scheduler) processPatch(patch *schedpb.SchedulePatch) { } for _, bfr := range patch.BackfillRequest { - startTime := timestamp.TimeValue(bfr.GetStartTime()) - // In previous versions the backfill start time was exclusive, ie when // the start time of the backfill matched the schedule's spec, it would // not be executed. This new version makes it inclusive instead. if s.hasMinVersion(InclusiveBackfillStartTime) { - startTime = startTime.Add(-1 * time.Millisecond) + startTime := timestamp.TimeValue(bfr.GetStartTime()).Add(-1 * time.Millisecond) + bfr.StartTime = timestamppb.New(startTime) + } + if s.hasMinVersion(IncrementalBackfill) { + // Add to ongoing backfills to process incrementally + if len(s.State.OngoingBackfills) >= s.tweakables.MaxBufferSize { + s.logger.Warn("Buffer overrun for backfill requests") + s.metrics.Counter(metrics.ScheduleBufferOverruns.Name()).Inc(1) + s.Info.BufferDropped += 1 + continue + } + s.State.OngoingBackfills = append(s.State.OngoingBackfills, common.CloneProto(bfr)) + } else { + // Old version: process whole backfill synchronously + s.processTimeRange( + timestamp.TimeValue(bfr.GetStartTime()), + timestamp.TimeValue(bfr.GetEndTime()), + bfr.GetOverlapPolicy(), + true, + nil, + ) } - - s.processTimeRange( - startTime, - timestamp.TimeValue(bfr.GetEndTime()), - bfr.GetOverlapPolicy(), - true, - ) } if patch.Pause != "" { @@ -578,6 +594,7 @@ func (s *scheduler) processTimeRange( t1, t2 time.Time, overlapPolicy enumspb.ScheduleOverlapPolicy, manual bool, + limit *int, ) time.Time { s.logger.Debug("processTimeRange", "t1", t1, "t2", t2, "overlap-policy", overlapPolicy, "manual", manual) @@ -624,6 +641,13 @@ func (s *scheduler) processTimeRange( continue } s.addStart(next.Nominal, next.Next, overlapPolicy, manual) + + if limit != nil { + (*limit)-- + if *limit <= 0 { + return t1 + } + } } } @@ -671,8 +695,11 @@ func (s *scheduler) sleep(nextWakeup time.Time) { forceCAN := workflow.GetSignalChannel(s.ctx, SignalNameForceCAN) sel.AddReceive(forceCAN, s.handleForceCANSignal) - // if we're paused or out of actions, we don't need to wake up until we get an update - if s.tweakables.SleepWhilePaused && !s.canTakeScheduledAction(false, false) { + if s.hasMoreAllowAllBackfills() { + // if we have more allow-all backfills to do, do a short sleep and continue + nextWakeup = s.now().Add(1 * time.Second) + } else if s.tweakables.SleepWhilePaused && !s.canTakeScheduledAction(false, false) { + // if we're paused or out of actions, we don't need to wake up until we get an update nextWakeup = time.Time{} } @@ -717,6 +744,38 @@ func (s *scheduler) sleep(nextWakeup time.Time) { } } +func (s *scheduler) processBackfills() { + limit := s.tweakables.BackfillsPerIteration + + for len(s.State.OngoingBackfills) > 0 && + limit > 0 && + // use only half the buffer for backfills + len(s.State.BufferedStarts) < s.tweakables.MaxBufferSize/2 { + bfr := s.State.OngoingBackfills[0] + startTime := timestamp.TimeValue(bfr.GetStartTime()) + endTime := timestamp.TimeValue(bfr.GetEndTime()) + next := s.processTimeRange( + startTime, + endTime, + bfr.GetOverlapPolicy(), + true, + &limit, + ) + if next.IsZero() || next.After(endTime) { + // done with this one + s.State.OngoingBackfills = s.State.OngoingBackfills[1:] + } else { + // adjust start time for next iteration + bfr.StartTime = timestamppb.New(next) + } + } +} + +func (s *scheduler) hasMoreAllowAllBackfills() bool { + return len(s.State.OngoingBackfills) > 0 && + s.resolveOverlapPolicy(s.State.OngoingBackfills[0].OverlapPolicy) == enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL +} + func (s *scheduler) wfWatcherReturned(f workflow.Future) { id := s.watchingWorkflowId s.watchingWorkflowId = "" @@ -1299,7 +1358,10 @@ func (s *scheduler) terminateWorkflow(ex *commonpb.WorkflowExecution) { func (s *scheduler) getRetentionExpiration(nextWakeup time.Time) time.Time { // if RetentionTime is not set or the schedule is paused or nextWakeup time is not zero // or there is more action to take, there is no need for retention - if s.tweakables.RetentionTime == 0 || s.Schedule.State.Paused || (!nextWakeup.IsZero() && s.canTakeScheduledAction(false, false)) { + if s.tweakables.RetentionTime == 0 || + s.Schedule.State.Paused || + (!nextWakeup.IsZero() && s.canTakeScheduledAction(false, false)) || + s.hasMoreAllowAllBackfills() { return time.Time{} } diff --git a/service/worker/scheduler/workflow_test.go b/service/worker/scheduler/workflow_test.go index de37d4de0cf..4ff7eefea7e 100644 --- a/service/worker/scheduler/workflow_test.go +++ b/service/worker/scheduler/workflow_test.go @@ -188,9 +188,10 @@ func (s *workflowSuite) expectTerminate(f func(req *schedspb.TerminateWorkflowRe // and delayed executions, which means they have to be supplied to runAcrossContinue as data. type workflowRun struct { - id string - start, end time.Time - result enumspb.WorkflowExecutionStatus + id string + start, end time.Time + startTolerance time.Duration + result enumspb.WorkflowExecutionStatus } type runAcrossContinueState struct { @@ -207,8 +208,8 @@ func (s *workflowSuite) setupMocksForWorkflows(runs []workflowRun, state *runAcr }) s.env.OnActivity(new(activities).StartWorkflow, mock.Anything, matchStart).Times(0).Maybe().Return( func(_ context.Context, req *schedspb.StartWorkflowRequest) (*schedspb.StartWorkflowResponse, error) { - if _, ok := state.started[req.Request.WorkflowId]; ok { - s.Failf("multiple starts for %s", req.Request.WorkflowId) + if prev, ok := state.started[req.Request.WorkflowId]; ok { + s.Failf("multiple starts", "for %s at %s (prev %s)", req.Request.WorkflowId, s.now(), prev) } state.started[req.Request.WorkflowId] = s.now() return &schedspb.StartWorkflowResponse{ @@ -237,6 +238,12 @@ func (s *workflowSuite) setupMocksForWorkflows(runs []workflowRun, state *runAcr return &schedspb.WatchWorkflowResponse{Status: run.result}, nil }) } + // catch unexpected starts + s.env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Times(0).Maybe().Return( + func(_ context.Context, req *schedspb.StartWorkflowRequest) (*schedspb.StartWorkflowResponse, error) { + s.Failf("unexpected start", "for %s at %s", req.Request.WorkflowId, s.now()) + return nil, nil + }) } type delayedCallback struct { @@ -314,7 +321,9 @@ func (s *workflowSuite) runAcrossContinue( // check starts that we actually got s.Require().Equal(len(runs), len(state.started)) for _, run := range runs { - s.Truef(run.start.Equal(state.started[run.id]), "%v != %v", run.start, state.started[run.id]) + actual := state.started[run.id] + inRange := !actual.Before(run.start.Add(-run.startTolerance)) && !actual.After(run.start.Add(run.startTolerance)) + s.Truef(inRange, "%v != %v for %s", run.start, state.started[run.id], run.id) } } } @@ -1311,6 +1320,141 @@ func (s *workflowSuite) TestBackfillInclusiveStartEnd() { ) } +func (s *workflowSuite) TestHugeBackfillAllowAll() { + // TODO: remove once default version is IncrementalBackfill + prevTweakables := currentTweakablePolicies + currentTweakablePolicies.Version = IncrementalBackfill + currentTweakablePolicies.MaxBufferSize = 30 // make smaller for testing + defer func() { currentTweakablePolicies = prevTweakables }() + + // This has been run for up to 5000, but it takes a very long time. Run only 100 normally. + const backfillRuns = 100 + const backfills = 4 + // The number that we process per iteration, with a 1s sleep per iteration, makes an + // effective "rate limit" for processing allow-all backfills. This is different from the + // explicit rate limit. + rateLimit := currentTweakablePolicies.BackfillsPerIteration + + base := time.Date(2001, 8, 6, 0, 0, 0, 0, time.UTC) + runs := make([]workflowRun, backfillRuns) + for i := range runs { + t := base.Add(time.Duration(i) * time.Hour) + actual := baseStartTime.Add(time.Minute).Add(time.Duration(i/rateLimit) * time.Second) + runs[i] = workflowRun{ + id: "myid-" + t.Format(time.RFC3339), + start: actual, + startTolerance: 5 * time.Second, // the "rate limit" isn't exact + end: actual.Add(time.Minute), + result: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + } + } + + delayedCallbacks := make([]delayedCallback, backfills) + for i := range delayedCallbacks { + i := i + delayedCallbacks[i] = delayedCallback{ + // test environment seems to get confused if the callback falls on the same instant + // as a workflow timer, so use an odd interval to force it to be different. + at: baseStartTime.Add(time.Minute).Add(time.Duration(i) * 1113 * time.Millisecond), + f: func() { + s.env.SignalWorkflow(SignalNamePatch, &schedpb.SchedulePatch{ + BackfillRequest: []*schedpb.BackfillRequest{{ + StartTime: timestamppb.New(base.Add(time.Duration(i*backfillRuns/backfills) * time.Hour)), + EndTime: timestamppb.New(base.Add(time.Duration((i+1)*backfillRuns/backfills-1) * time.Hour)), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }}, + }) + }, + } + } + + delayedCallbacks = append(delayedCallbacks, delayedCallback{ + at: baseStartTime.Add(50 * time.Minute), + finishTest: true, + }) + + s.runAcrossContinue( + runs, + delayedCallbacks, + &schedpb.Schedule{ + Spec: &schedpb.ScheduleSpec{ + Interval: []*schedpb.IntervalSpec{{Interval: durationpb.New(time.Hour)}}, + }, + }, + ) +} + +func (s *workflowSuite) TestHugeBackfillBuffer() { + // TODO: remove once default version is IncrementalBackfill + prevTweakables := currentTweakablePolicies + currentTweakablePolicies.Version = IncrementalBackfill + currentTweakablePolicies.MaxBufferSize = 30 // make smaller for testing + defer func() { currentTweakablePolicies = prevTweakables }() + + // This has been run for up to 3000, but it takes a very long time. Run only 100 normally. + const backfillRuns = 100 + const backfills = 4 + + base := time.Date(2001, 8, 6, 0, 0, 0, 0, time.UTC) + runs := make([]workflowRun, backfillRuns) + duration := 56 * time.Second + for i := range runs { + t := base.Add(time.Duration(i) * time.Hour) + actual := baseStartTime.Add(time.Minute).Add(time.Duration(i) * duration) + runs[i] = workflowRun{ + id: "myid-" + t.Format(time.RFC3339), + start: actual, + end: actual.Add(duration), + result: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + } + } + testEnd := baseStartTime.Add(time.Minute).Add(time.Duration(backfillRuns) * duration) + // also add normal runs during the time it takes them all to run + for t := baseStartTime.Add(time.Hour); t.Before(testEnd); t = t.Add(time.Hour) { + runs = append(runs, workflowRun{ + id: "myid-" + t.Format(time.RFC3339), + start: t, + end: t.Add(duration), + result: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + }) + } + + delayedCallbacks := make([]delayedCallback, backfills) + for i := range delayedCallbacks { + i := i + delayedCallbacks[i] = delayedCallback{ + at: baseStartTime.Add(time.Minute).Add(time.Duration(i) * 1113 * time.Millisecond), + f: func() { + s.env.SignalWorkflow(SignalNamePatch, &schedpb.SchedulePatch{ + BackfillRequest: []*schedpb.BackfillRequest{{ + StartTime: timestamppb.New(base.Add(time.Duration(i*backfillRuns/backfills) * time.Hour)), + EndTime: timestamppb.New(base.Add(time.Duration((i+1)*backfillRuns/backfills-1) * time.Hour)), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + }}, + }) + }, + } + } + + delayedCallbacks = append(delayedCallbacks, delayedCallback{ + at: testEnd, + finishTest: true, + }) + + s.runAcrossContinue( + runs, + delayedCallbacks, + &schedpb.Schedule{ + Spec: &schedpb.ScheduleSpec{ + Interval: []*schedpb.IntervalSpec{{Interval: durationpb.New(time.Hour)}}, + }, + Policies: &schedpb.SchedulePolicies{ + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }, + }, + ) +} + func (s *workflowSuite) TestPause() { s.runAcrossContinue( []workflowRun{