From fead819d61e2d0dcbb1f73b99df714476f8856b1 Mon Sep 17 00:00:00 2001 From: iamlinjunhong <1030420200@qq.com> Date: Tue, 9 Jun 2026 17:37:01 +0800 Subject: [PATCH 1/6] fix(lockservice): fence stale binds by allocator epoch --- pkg/lockservice/lock_table_allocator.go | 8 +- pkg/lockservice/lock_table_keeper.go | 22 +- pkg/lockservice/lock_table_remote.go | 2 +- pkg/lockservice/log.go | 40 +++ pkg/lockservice/service.go | 112 +++++++- pkg/lockservice/service_remote.go | 6 +- pkg/lockservice/service_test.go | 339 +++++++++++++++++++++++- pkg/pb/lock/lock.pb.go | 339 ++++++++++++++---------- pkg/util/metric/v2/lockservice.go | 70 +++++ pkg/util/metric/v2/metrics.go | 1 + proto/lock.proto | 4 +- 11 files changed, 778 insertions(+), 165 deletions(-) create mode 100644 pkg/util/metric/v2/lockservice.go diff --git a/pkg/lockservice/lock_table_allocator.go b/pkg/lockservice/lock_table_allocator.go index c5259236a7341..974b6bf022326 100644 --- a/pkg/lockservice/lock_table_allocator.go +++ b/pkg/lockservice/lock_table_allocator.go @@ -40,8 +40,10 @@ type lockTableAllocator struct { client Client inactiveService sync.Map // lock service id -> inactive time ctl sync.Map // lock service id -> *commitCtl - version uint64 - mu struct { + // version is the allocator process epoch. It is set once when the + // allocator is constructed; production code must not mutate it at runtime. + version uint64 + mu struct { sync.RWMutex services map[string]*serviceBinds lockTables map[uint32]map[uint64]pb.LockTable @@ -843,6 +845,7 @@ func (l *lockTableAllocator) handleGetBind( req *pb.Request, resp *pb.Response, cs morpc.ClientSession) { + resp.GetBind.AllocatorVersion = l.version if !l.canGetBind(req.GetBind.ServiceID) { writeResponse(l.logger, cancel, resp, moerr.NewNewTxnInCNRollingRestart(), cs) return @@ -862,6 +865,7 @@ func (l *lockTableAllocator) handleKeepLockTableBind( req *pb.Request, resp *pb.Response, cs morpc.ClientSession) { + resp.KeepLockTableBind.AllocatorVersion = l.version resp.KeepLockTableBind.OK = l.KeepLockTableBind(req.KeepLockTableBind.ServiceID) if !resp.KeepLockTableBind.OK { // resp.KeepLockTableBind.Status = pb.Status_ServiceCanRestart diff --git a/pkg/lockservice/lock_table_keeper.go b/pkg/lockservice/lock_table_keeper.go index d14fb0a4e1d93..05da36d3b607a 100644 --- a/pkg/lockservice/lock_table_keeper.go +++ b/pkg/lockservice/lock_table_keeper.go @@ -196,6 +196,10 @@ func (k *lockTableKeeper) doKeepLockTableBind(ctx context.Context) { defer releaseResponse(resp) if resp.KeepLockTableBind.OK { + k.service.observeAllocatorVersionWithHolders( + "keepalive", + resp.KeepLockTableBind.AllocatorVersion, + k.groupTables) switch resp.KeepLockTableBind.Status { case pb.Status_ServiceLockEnable: if !k.service.isStatus(pb.Status_ServiceLockEnable) { @@ -219,19 +223,11 @@ func (k *lockTableKeeper) doKeepLockTableBind(ctx context.Context) { return } - n := 0 - k.groupTables.removeWithFilter(func(_ uint64, v lockTable) bool { - newVersion := k.groupTables.getVersion() - if oldVersion != newVersion { - return false - } - bind := v.getBind() - if bind.ServiceID == k.serviceID { - n++ - return true - } - return false - }) + n := k.service.handleKeepBindFailed( + k.serviceID, + k.groupTables, + oldVersion, + resp.KeepLockTableBind.AllocatorVersion) if n > 0 { // Keep bind receiving an explicit failure means that all the binds of the local diff --git a/pkg/lockservice/lock_table_remote.go b/pkg/lockservice/lock_table_remote.go index 089be9ef4f118..b7fd085037b0e 100644 --- a/pkg/lockservice/lock_table_remote.go +++ b/pkg/lockservice/lock_table_remote.go @@ -339,7 +339,7 @@ func (l *remoteLockTable) handleError( // Note. Since the cn where the remote lock table is located may // be permanently gone, we need to go to the allocator to check if // the bind is valid. - new, err := getLockTableBind( + new, _, err := getLockTableBind( l.client, l.bind.Group, l.bind.Table, diff --git a/pkg/lockservice/log.go b/pkg/lockservice/log.go index 3cfa19cb4ecb6..a5bd62ec73dae 100644 --- a/pkg/lockservice/log.go +++ b/pkg/lockservice/log.go @@ -473,6 +473,46 @@ func logLockAllocatorStartSucc( ) } +func logAllocatorEpochChanged( + logger *log.MOLogger, + source string, + oldVersion uint64, + newVersion uint64, + removed int, +) { + if logger == nil { + return + } + + logger.Log( + "lock allocator epoch changed", + getLogOptions(zap.InfoLevel), + zap.String("source", source), + zap.Uint64("old-version", oldVersion), + zap.Uint64("new-version", newVersion), + zap.Int("removed", removed), + ) +} + +func logAllocatorEpochRegression( + logger *log.MOLogger, + source string, + oldVersion uint64, + observedVersion uint64, +) { + if logger == nil { + return + } + + logger.Log( + "lock allocator epoch regression observed", + getLogOptions(zap.WarnLevel), + zap.String("source", source), + zap.Uint64("old-version", oldVersion), + zap.Uint64("observed-version", observedVersion), + ) +} + func logCheckDeadLockFailed( logger *log.MOLogger, waitingTxn, txn pb.WaitTxn, diff --git a/pkg/lockservice/service.go b/pkg/lockservice/service.go index f480ae1c4b650..07f2c0ea69b91 100644 --- a/pkg/lockservice/service.go +++ b/pkg/lockservice/service.go @@ -60,6 +60,9 @@ type service struct { fetchWhoWaitingListC chan who logger *log.MOLogger + allocatorVersionMu sync.Mutex + lastAllocatorVersion uint64 + remote struct { client Client server Server @@ -585,7 +588,7 @@ func (s *service) getLockTableWithCreate( close(c) }() - bind, err := getLockTableBind( + bind, allocatorVersion, err := getLockTableBind( s.remote.client, group, tableID, @@ -596,6 +599,7 @@ func (s *service) getLockTableWithCreate( return nil, err } + s.observeAllocatorVersion("get-bind", allocatorVersion) v := s.tableGroups.set(group, tableID, s.createLockTableByBind(bind)) return v, nil } @@ -605,6 +609,95 @@ func (s *service) handleBindChanged(newBind pb.LockTable) { s.tableGroups.set(newBind.Group, newBind.Table, new) } +func (s *service) observeAllocatorVersion(source string, observedVersion uint64) int { + return s.observeAllocatorVersionWithHolders(source, observedVersion, s.tableGroups) +} + +func (s *service) observeAllocatorVersionWithHolders( + source string, + observedVersion uint64, + holders *lockTableHolders, +) int { + if observedVersion == 0 { + return 0 + } + + s.allocatorVersionMu.Lock() + defer s.allocatorVersionMu.Unlock() + return s.observeAllocatorVersionLocked(source, observedVersion, holders) +} + +func (s *service) observeAllocatorVersionLocked( + source string, + observedVersion uint64, + holders *lockTableHolders, +) int { + if observedVersion == 0 { + return 0 + } + + oldVersion := s.lastAllocatorVersion + if oldVersion != 0 && observedVersion < oldVersion { + v2.GetLockServiceAllocatorEpochRegressionCounter(source).Inc() + logAllocatorEpochRegression(s.logger, source, oldVersion, observedVersion) + return 0 + } + if observedVersion == oldVersion { + return 0 + } + + // Defensive path for minimal service instances used by keeper tests. + // Normal lock services always pass a holder and can purge stale binds. + if holders == nil { + s.lastAllocatorVersion = observedVersion + v2.LockServiceAllocatorEpochObservedGauge.Set(float64(observedVersion)) + v2.GetLockServiceAllocatorEpochChangedCounter(source).Inc() + logAllocatorEpochChanged(s.logger, source, oldVersion, observedVersion, 0) + return 0 + } + + removed := holders.removeWithFilter(func(_ uint64, lt lockTable) bool { + return lt.getBind().Version < observedVersion + }) + s.lastAllocatorVersion = observedVersion + v2.LockServiceAllocatorEpochObservedGauge.Set(float64(observedVersion)) + v2.GetLockServiceAllocatorEpochChangedCounter(source).Inc() + if removed > 0 { + v2.GetLockServiceStaleBindPurgedCounter(source).Add(float64(removed)) + } + logAllocatorEpochChanged(s.logger, source, oldVersion, observedVersion, removed) + return removed +} + +func (s *service) handleKeepBindFailed( + serviceID string, + holders *lockTableHolders, + oldTableVersion, + allocatorVersion uint64, +) int { + s.allocatorVersionMu.Lock() + defer s.allocatorVersionMu.Unlock() + + if holders == nil { + s.observeAllocatorVersionLocked("keepalive-ok-false-epoch", allocatorVersion, nil) + return 0 + } + + // Keep the original OK=false snapshot guard: if another local purge already + // changed holders.version, this service-level purge should skip this round. + removed := holders.removeWithFilter(func(_ uint64, lt lockTable) bool { + if oldTableVersion != holders.getVersion() { + return false + } + return lt.getBind().ServiceID == serviceID + }) + if removed > 0 { + v2.GetLockServiceStaleBindPurgedCounter("keepalive-ok-false-service").Add(float64(removed)) + } + s.observeAllocatorVersionLocked("keepalive-ok-false-epoch", allocatorVersion, holders) + return removed +} + func (s *service) createLockTableByBind(bind pb.LockTable) lockTable { defer logLockTableCreated( s.logger, @@ -1065,19 +1158,18 @@ func (m *lockTableHolders) iter(fn func(uint64, lockTable) bool) { } } -func (m *lockTableHolders) removeWithFilter(filter func(uint64, lockTable) bool) { +func (m *lockTableHolders) removeWithFilter(filter func(uint64, lockTable) bool) int { m.RLock() defer m.RUnlock() - removed := false + removed := 0 for _, h := range m.holders { - if h.removeWithFilter(filter) { - removed = true - } + removed += h.removeWithFilter(filter) } - if removed { + if removed > 0 { m.version.Add(1) } + return removed } // getVersion returns the current version of the lockTableHolders @@ -1135,15 +1227,15 @@ func (m *lockTableHolder) iter(fn func(uint64, lockTable) bool) bool { return true } -func (m *lockTableHolder) removeWithFilter(filter func(uint64, lockTable) bool) bool { +func (m *lockTableHolder) removeWithFilter(filter func(uint64, lockTable) bool) int { m.Lock() defer m.Unlock() - removed := false + removed := 0 for id, v := range m.tables { if filter(id, v) { v.close() delete(m.tables, id) - removed = true + removed++ } } return removed diff --git a/pkg/lockservice/service_remote.go b/pkg/lockservice/service_remote.go index 05821f75cda76..103c57d8a200b 100644 --- a/pkg/lockservice/service_remote.go +++ b/pkg/lockservice/service_remote.go @@ -639,7 +639,7 @@ func getLockTableBind( tableID uint64, originTableID uint64, serviceID string, - sharding pb.Sharding) (pb.LockTable, error) { + sharding pb.Sharding) (pb.LockTable, uint64, error) { ctx, cancel := context.WithTimeoutCause(context.Background(), defaultRPCTimeout, moerr.CauseGetLockTableBind) defer cancel() @@ -655,11 +655,11 @@ func getLockTableBind( resp, err := c.Send(ctx, req) if err != nil { - return pb.LockTable{}, moerr.AttachCause(ctx, err) + return pb.LockTable{}, 0, moerr.AttachCause(ctx, err) } defer releaseResponse(resp) v := resp.GetBind.LockTable - return v, nil + return v, resp.GetBind.AllocatorVersion, nil } type who struct { diff --git a/pkg/lockservice/service_test.go b/pkg/lockservice/service_test.go index 96616f1d72cdb..ecea5843f529e 100644 --- a/pkg/lockservice/service_test.go +++ b/pkg/lockservice/service_test.go @@ -2931,7 +2931,7 @@ func TestLeaveGetBindInRollingRestartCN(t *testing.T) { } } // get bind - _, err = getLockTableBind( + _, _, err = getLockTableBind( l.remote.client, 0, 0, @@ -3213,6 +3213,343 @@ func TestResumeInvalidService(t *testing.T) { ) } +func TestCommitDetectsStaleLocalBindAfterAllocatorRestart(t *testing.T) { + runLockServiceTestsWithLevel( + t, + zapcore.DebugLevel, + []string{"s1"}, + time.Hour, + func(alloc *lockTableAllocator, s []*service) { + l1 := s[0] + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + staleTable := uint64(272592) + freshTable := uint64(50011829) + option := newTestRowExclusiveOptions() + + _, err := l1.Lock(ctx, staleTable, newTestRows(1), newTestTxnID(1), option) + require.NoError(t, err) + require.NoError(t, l1.Unlock(ctx, newTestTxnID(1), timestamp.Timestamp{})) + + staleLT := l1.tableGroups.get(0, staleTable) + require.NotNil(t, staleLT) + staleBind := staleLT.getBind() + require.Equal(t, l1.serviceID, staleBind.ServiceID) + + // Simulate a DN/TN pod rebuild: the allocator gets a new epoch and + // loses its in-memory bind map, while the CN service keeps running + // with the old local lock table cached. + alloc.mu.Lock() + alloc.mu.services = make(map[string]*serviceBinds) + alloc.mu.lockTables = make(map[uint32]map[uint64]pb.LockTable) + alloc.version++ + restartedVersion := alloc.version + alloc.mu.Unlock() + + require.Equal(t, staleBind, l1.tableGroups.get(0, staleTable).getBind()) + + invalid, err := alloc.Valid( + l1.serviceID, + []byte("commit-txn-before-refresh"), + []pb.LockTable{staleBind}) + require.NoError(t, err) + require.Equal(t, []uint64{staleTable}, invalid) + + // Touching another table registers the same CN service in the new + // allocator epoch. The response-level allocator version should make + // the CN purge the stale local table before caching the fresh bind. + _, err = l1.Lock(ctx, freshTable, newTestRows(2), newTestTxnID(2), option) + require.NoError(t, err) + require.NoError(t, l1.Unlock(ctx, newTestTxnID(2), timestamp.Timestamp{})) + + require.Nil(t, l1.tableGroups.get(0, staleTable)) + freshBind := l1.tableGroups.get(0, freshTable).getBind() + require.Equal(t, l1.serviceID, freshBind.ServiceID) + require.Equal(t, restartedVersion, freshBind.Version) + require.True(t, alloc.KeepLockTableBind(l1.serviceID)) + + alloc.mu.Lock() + _, ok := alloc.getLockTablesLocked(0)[staleTable] + alloc.mu.Unlock() + require.False(t, ok) + + _, err = l1.Lock(ctx, staleTable, newTestRows(3), newTestTxnID(3), option) + require.NoError(t, err) + refreshedBind := l1.tableGroups.get(0, staleTable).getBind() + require.True(t, refreshedBind.Changed(staleBind)) + require.Equal(t, restartedVersion, refreshedBind.Version) + require.NoError(t, l1.Unlock(ctx, newTestTxnID(3), timestamp.Timestamp{})) + }, + nil, + ) +} + +func TestAllocatorVersionZeroKeepsLocalBinds(t *testing.T) { + runLockServiceTests( + t, + []string{"s1"}, + func(_ *lockTableAllocator, s []*service) { + l1 := s[0] + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + table := uint64(14039) + option := newTestRowExclusiveOptions() + _, err := l1.Lock(ctx, table, newTestRows(1), newTestTxnID(1), option) + require.NoError(t, err) + require.NoError(t, l1.Unlock(ctx, newTestTxnID(1), timestamp.Timestamp{})) + + lt := l1.tableGroups.get(0, table) + require.NotNil(t, lt) + bind := lt.getBind() + lastVersion := l1.lastAllocatorVersion + + removed := l1.observeAllocatorVersion("compat-test", 0) + require.Zero(t, removed) + require.Equal(t, lastVersion, l1.lastAllocatorVersion) + require.Equal(t, bind, l1.tableGroups.get(0, table).getBind()) + }, + ) +} + +func TestAllocatorObserverDoesNotPurgeSameEpochBindVersions(t *testing.T) { + runLockServiceTests( + t, + []string{"s1"}, + func(_ *lockTableAllocator, s []*service) { + l1 := s[0] + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + tableA := uint64(18001) + tableB := uint64(18002) + option := newTestRowExclusiveOptions() + _, err := l1.Lock(ctx, tableA, newTestRows(1), newTestTxnID(1), option) + require.NoError(t, err) + require.NoError(t, l1.Unlock(ctx, newTestTxnID(1), timestamp.Timestamp{})) + + bindA := l1.tableGroups.get(0, tableA).getBind() + bindB := bindA + bindB.Table = tableB + bindB.OriginTable = tableB + bindB.Version = bindA.Version + 1 + l1.tableGroups.set(0, tableB, l1.createLockTableByBind(bindB)) + + l1.allocatorVersionMu.Lock() + l1.lastAllocatorVersion = 0 + l1.allocatorVersionMu.Unlock() + + removed := l1.observeAllocatorVersion("same-epoch-test", bindA.Version) + require.Zero(t, removed) + require.Equal(t, bindA, l1.tableGroups.get(0, tableA).getBind()) + require.Equal(t, bindB, l1.tableGroups.get(0, tableB).getBind()) + }, + ) +} + +func TestGetBindAllowsRegressedAllocatorVersionWithoutPurging(t *testing.T) { + runLockServiceTests( + t, + []string{"s1"}, + func(alloc *lockTableAllocator, s []*service) { + l1 := s[0] + table := uint64(19001) + + l1.allocatorVersionMu.Lock() + l1.lastAllocatorVersion = alloc.version + 1 + lastAllocatorVersion := l1.lastAllocatorVersion + l1.allocatorVersionMu.Unlock() + + _, err := l1.getLockTableWithCreate(0, table, newTestRows(1), pb.Sharding_None) + require.NoError(t, err) + require.Equal(t, lastAllocatorVersion, l1.lastAllocatorVersion) + require.Equal(t, alloc.version, l1.tableGroups.get(0, table).getBind().Version) + }, + ) +} + +func TestKeepaliveEpochPurgeKeepsGroupMovePop(t *testing.T) { + runLockServiceTests( + t, + []string{"s1"}, + func(alloc *lockTableAllocator, s []*service) { + l1 := s[0] + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + table := uint64(21001) + option := newTestRowExclusiveOptions() + _, err := l1.Lock(ctx, table, newTestRows(1), newTestTxnID(1), option) + require.NoError(t, err) + require.NoError(t, l1.Unlock(ctx, newTestTxnID(1), timestamp.Timestamp{})) + require.NotNil(t, l1.tableGroups.get(0, table)) + + l1.checkCanMoveGroupTables() + require.NotNil(t, l1.topGroupTables()) + + alloc.mu.Lock() + alloc.version++ + restartedVersion := alloc.version + alloc.mu.Unlock() + + l1.remote.keeper.(*lockTableKeeper).doKeepLockTableBind(ctx) + + require.Nil(t, l1.topGroupTables()) + require.Nil(t, l1.tableGroups.get(0, table)) + require.Equal(t, restartedVersion, l1.lastAllocatorVersion) + }, + ) +} + +func TestAllocatorObserverPurgesRemoteAndProxyLockTables(t *testing.T) { + runLockServiceTestsWithAdjustConfig( + t, + []string{"s1"}, + time.Second*10, + func(_ *lockTableAllocator, s []*service) { + l1 := s[0] + allocatorVersion := uint64(100) + remoteTable := uint64(22001) + proxyTable := uint64(22002) + + remoteBind := pb.LockTable{ + Group: 0, + Table: remoteTable, + OriginTable: remoteTable, + ServiceID: "s2", + Version: allocatorVersion - 1, + Valid: true, + } + l1.tableGroups.set( + 0, + remoteTable, + newRemoteLockTable( + l1.serviceID, + time.Second, + remoteBind, + l1.remote.client, + l1.handleBindChanged, + l1.logger)) + + proxyBind := remoteBind + proxyBind.Table = proxyTable + proxyBind.OriginTable = proxyTable + proxy := l1.createLockTableByBind(proxyBind) + _, ok := proxy.(*localLockTableProxy) + require.True(t, ok) + l1.tableGroups.set(0, proxyTable, proxy) + + removed := l1.observeAllocatorVersion("remote-proxy-test", allocatorVersion) + require.Equal(t, 2, removed) + require.Nil(t, l1.tableGroups.get(0, remoteTable)) + require.Nil(t, l1.tableGroups.get(0, proxyTable)) + }, + func(cfg *Config) { + cfg.EnableRemoteLocalProxy = true + }) +} + +func TestAllocatorObserverConcurrentKeepaliveAndGetBind(t *testing.T) { + runLockServiceTests( + t, + []string{"s1"}, + func(alloc *lockTableAllocator, s []*service) { + l1 := s[0] + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + staleTable := uint64(23001) + freshTable := uint64(23002) + option := newTestRowExclusiveOptions() + _, err := l1.Lock(ctx, staleTable, newTestRows(1), newTestTxnID(1), option) + require.NoError(t, err) + require.NoError(t, l1.Unlock(ctx, newTestTxnID(1), timestamp.Timestamp{})) + require.NotNil(t, l1.tableGroups.get(0, staleTable)) + + alloc.mu.Lock() + alloc.mu.services = make(map[string]*serviceBinds) + alloc.mu.lockTables = make(map[uint32]map[uint64]pb.LockTable) + alloc.version++ + restartedVersion := alloc.version + alloc.mu.Unlock() + + var wg sync.WaitGroup + errC := make(chan error, 1) + wg.Add(3) + go func() { + defer wg.Done() + l1.observeAllocatorVersion("concurrent-test", restartedVersion) + }() + go func() { + defer wg.Done() + l1.remote.keeper.(*lockTableKeeper).doKeepLockTableBind(ctx) + }() + go func() { + defer wg.Done() + _, err := l1.Lock(ctx, freshTable, newTestRows(2), newTestTxnID(2), option) + if err != nil { + errC <- err + return + } + if err := l1.Unlock(ctx, newTestTxnID(2), timestamp.Timestamp{}); err != nil { + errC <- err + } + }() + wg.Wait() + close(errC) + require.NoError(t, <-errC) + require.Nil(t, l1.tableGroups.get(0, staleTable)) + require.Equal(t, restartedVersion, l1.lastAllocatorVersion) + }, + ) +} + +func TestAllocatorObserverCloseWaitersOnStaleLocalBind(t *testing.T) { + runLockServiceTests( + t, + []string{"s1"}, + func(_ *lockTableAllocator, s []*service) { + l1 := s[0] + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + table := uint64(24001) + rows := newTestRows(1) + txn1 := newTestTxnID(1) + txn2 := newTestTxnID(2) + option := newTestRowExclusiveOptions() + + _, err := l1.Lock(ctx, table, rows, txn1, option) + require.NoError(t, err) + staleLT := l1.tableGroups.get(0, table) + require.NotNil(t, staleLT) + staleBind := staleLT.getBind() + + errC := make(chan error, 1) + go func() { + _, err := l1.Lock(ctx, table, rows, txn2, option) + errC <- err + }() + require.NoError(t, waitLocalWaiters(staleLT.(*localLockTable), rows[0], 1)) + + removed := l1.observeAllocatorVersion("waiter-close-test", staleBind.Version+1) + require.Equal(t, 1, removed) + require.Nil(t, l1.tableGroups.get(0, table)) + + select { + case err := <-errC: + require.True(t, moerr.IsMoErrCode(err, moerr.ErrLockTableNotFound), err) + case <-ctx.Done(): + t.Fatal("waiter was not notified by stale bind purge") + } + + require.NoError(t, l1.Unlock(ctx, txn1, timestamp.Timestamp{})) + require.NoError(t, l1.Unlock(ctx, txn2, timestamp.Timestamp{})) + }, + ) +} + func TestRetryLockSuccInRollingRestartCN(t *testing.T) { runLockServiceTests( t, diff --git a/pkg/pb/lock/lock.pb.go b/pkg/pb/lock/lock.pb.go index 9e734a9c9c139..fb342736ca2ca 100644 --- a/pkg/pb/lock/lock.pb.go +++ b/pkg/pb/lock/lock.pb.go @@ -1454,6 +1454,7 @@ func (m *GetBindRequest) GetOriginTable() uint64 { // GetBindResponse get bind request from allocator response. TN -> CN type GetBindResponse struct { LockTable LockTable `protobuf:"bytes,2,opt,name=LockTable,proto3" json:"LockTable"` + AllocatorVersion uint64 `protobuf:"varint,3,opt,name=AllocatorVersion,proto3" json:"AllocatorVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1499,6 +1500,13 @@ func (m *GetBindResponse) GetLockTable() LockTable { return LockTable{} } +func (m *GetBindResponse) GetAllocatorVersion() uint64 { + if m != nil { + return m.AllocatorVersion + } + return 0 +} + // KeepLockTableBindRequest keep lock table bind request. CN -> TN type KeepLockTableBindRequest struct { ServiceID string `protobuf:"bytes,1,opt,name=ServiceID,proto3" json:"ServiceID,omitempty"` @@ -1575,6 +1583,7 @@ func (m *KeepLockTableBindRequest) GetLockTables() []LockTable { type KeepLockTableBindResponse struct { OK bool `protobuf:"varint,1,opt,name=OK,proto3" json:"OK,omitempty"` Status Status `protobuf:"varint,2,opt,name=Status,proto3,enum=lock.Status" json:"Status,omitempty"` + AllocatorVersion uint64 `protobuf:"varint,3,opt,name=AllocatorVersion,proto3" json:"AllocatorVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1627,6 +1636,13 @@ func (m *KeepLockTableBindResponse) GetStatus() Status { return Status_ServiceLockEnable } +func (m *KeepLockTableBindResponse) GetAllocatorVersion() uint64 { + if m != nil { + return m.AllocatorVersion + } + return 0 +} + // SetRestartServiceRequest set restart lock service request from allocator request. operator -> TN type SetRestartServiceRequest struct { ServiceID string `protobuf:"bytes,1,opt,name=ServiceID,proto3" json:"ServiceID,omitempty"` @@ -2889,140 +2905,141 @@ func init() { func init() { proto.RegisterFile("lock.proto", fileDescriptor_164ad2988c7acaf1) } var fileDescriptor_164ad2988c7acaf1 = []byte{ - // 2113 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x59, 0x4d, 0x73, 0x1b, 0xc7, - 0xd1, 0xe6, 0x02, 0x20, 0x3e, 0x1a, 0x00, 0xb9, 0x1c, 0x52, 0xd4, 0x92, 0xaf, 0x4d, 0xe1, 0xdd, - 0xd0, 0x55, 0x30, 0x1d, 0x8b, 0x11, 0x15, 0xd9, 0x8e, 0x9c, 0xa8, 0x42, 0x81, 0x22, 0x4d, 0x4b, - 0x14, 0x9d, 0x01, 0xa4, 0x54, 0xe5, 0xb6, 0x04, 0x46, 0xe4, 0x16, 0xc1, 0x5d, 0x64, 0xb1, 0x20, - 0xc1, 0x1f, 0x90, 0xaa, 0x54, 0xe5, 0x9e, 0x63, 0x2a, 0x47, 0x5f, 0xf2, 0x3f, 0x7c, 0x74, 0x55, - 0xee, 0xf9, 0x50, 0xfe, 0x48, 0xaa, 0x67, 0x66, 0xb1, 0x33, 0xfb, 0x41, 0xd8, 0xb9, 0xed, 0x74, - 0xf7, 0x3c, 0xdd, 0xd3, 0x33, 0xfb, 0x4c, 0xf7, 0x2e, 0xc0, 0xd0, 0xef, 0x5f, 0x3e, 0x1c, 0x05, - 0x7e, 0xe8, 0x93, 0x12, 0x3e, 0x6f, 0x7e, 0x7a, 0xee, 0x86, 0x17, 0x93, 0xb3, 0x87, 0x7d, 0xff, - 0x6a, 0xf7, 0xdc, 0x3f, 0xf7, 0x77, 0xb9, 0xf2, 0x6c, 0xf2, 0x8e, 0x8f, 0xf8, 0x80, 0x3f, 0x89, - 0x49, 0x9b, 0xcb, 0xa1, 0x7b, 0xc5, 0xc6, 0xa1, 0x73, 0x35, 0x12, 0x02, 0xfb, 0x2f, 0x45, 0xa8, - 0xbf, 0xf2, 0xfb, 0x97, 0xa7, 0xa3, 0xd0, 0xf5, 0xbd, 0x31, 0x79, 0x0c, 0xf5, 0xa3, 0xc0, 0xf1, - 0x26, 0x43, 0x27, 0x70, 0xc3, 0x5b, 0xcb, 0x68, 0x19, 0xed, 0xa5, 0xbd, 0x95, 0x87, 0xdc, 0xaf, - 0xa2, 0xa0, 0xaa, 0x15, 0xb1, 0xa1, 0x74, 0xe2, 0x0f, 0x98, 0x55, 0xe0, 0xd6, 0x4b, 0xc2, 0x1a, - 0x51, 0x51, 0x4a, 0xb9, 0x8e, 0xb4, 0xa1, 0xfc, 0x8d, 0x3f, 0x74, 0xfb, 0xb7, 0x56, 0x91, 0x5b, - 0x99, 0xc2, 0xea, 0xb7, 0x8e, 0x1b, 0x0a, 0x39, 0x95, 0x7a, 0xf2, 0x01, 0xd4, 0x0e, 0xfd, 0xe0, - 0xc6, 0x09, 0x06, 0x3d, 0xdf, 0x2a, 0xb5, 0x8c, 0x76, 0x8d, 0xc6, 0x02, 0xd2, 0x86, 0xe5, 0x9e, - 0x73, 0x36, 0x64, 0x07, 0xec, 0x5d, 0xe7, 0xc2, 0xf1, 0xce, 0xd9, 0xc0, 0x5a, 0x6c, 0x19, 0xed, - 0x2a, 0x4d, 0x8a, 0x11, 0x87, 0xb2, 0x30, 0xb8, 0x45, 0x17, 0x56, 0xb9, 0x65, 0xb4, 0x8b, 0x34, - 0x16, 0x90, 0x35, 0x58, 0x3c, 0x0a, 0xfc, 0xc9, 0xc8, 0xaa, 0xb4, 0x8c, 0x76, 0x93, 0x8a, 0x01, - 0xd9, 0x81, 0x6a, 0xf7, 0xc2, 0x09, 0x06, 0xae, 0x77, 0x6e, 0x55, 0xd5, 0xd5, 0x44, 0x52, 0x3a, - 0xd3, 0x93, 0xa7, 0x00, 0x5d, 0xcf, 0x19, 0x75, 0x2f, 0xfc, 0xb0, 0x37, 0xb6, 0x6a, 0x2d, 0xa3, - 0x5d, 0xdf, 0x5b, 0x7b, 0x18, 0x27, 0xb8, 0x17, 0x3d, 0x3d, 0x2f, 0x7d, 0xf7, 0x8f, 0x07, 0x0b, - 0x54, 0xb1, 0xc6, 0x55, 0x60, 0x7e, 0x30, 0x12, 0x34, 0xf3, 0x27, 0xa1, 0x05, 0x3c, 0xc2, 0xa4, - 0xd8, 0xfe, 0xbb, 0x01, 0x35, 0x94, 0xf1, 0xd5, 0x61, 0xd4, 0xfc, 0x81, 0x6f, 0x4c, 0x89, 0x8a, - 0x01, 0xae, 0xb4, 0xcb, 0x82, 0x6b, 0xb7, 0xcf, 0x8e, 0x0f, 0xf8, 0x26, 0xd4, 0x68, 0x2c, 0x20, - 0x16, 0x54, 0xde, 0xb2, 0x60, 0xec, 0xfa, 0x1e, 0x4f, 0x7d, 0x89, 0x46, 0x43, 0x44, 0x7b, 0xeb, - 0x0c, 0xdd, 0x01, 0xcf, 0x72, 0x95, 0x8a, 0x41, 0x9c, 0x99, 0xc5, 0xbc, 0xcc, 0x94, 0xe7, 0x64, - 0xa6, 0x05, 0xf5, 0xd3, 0xc0, 0x3d, 0x77, 0x3d, 0x11, 0x6b, 0x85, 0x7b, 0x55, 0x45, 0xf6, 0x9f, - 0x00, 0x2a, 0x94, 0xfd, 0x7e, 0xc2, 0xc6, 0xa1, 0xd8, 0x27, 0xfe, 0x78, 0x7c, 0x20, 0xd7, 0x15, - 0x0b, 0xc8, 0x63, 0x65, 0xf9, 0x7c, 0x6d, 0xf5, 0xbd, 0xe5, 0xf8, 0x80, 0x71, 0xb1, 0xcc, 0xaf, - 0x92, 0xa6, 0x6d, 0x28, 0x9f, 0xb0, 0xf0, 0xc2, 0x1f, 0xc8, 0xc3, 0xd6, 0x10, 0x33, 0x84, 0x8c, - 0x4a, 0x1d, 0xf9, 0x04, 0x4a, 0x38, 0x85, 0xaf, 0xbe, 0x1e, 0x1d, 0x72, 0x94, 0x48, 0xef, 0x12, - 0x97, 0x1b, 0x91, 0x47, 0x50, 0x7e, 0xe3, 0xa1, 0x05, 0x4f, 0x4b, 0x7d, 0x6f, 0x55, 0x98, 0x0b, - 0x99, 0x3e, 0x41, 0x1a, 0x92, 0x5f, 0x01, 0x1c, 0xb1, 0xb0, 0x37, 0xf5, 0xb8, 0x97, 0x32, 0x9f, - 0x76, 0x5f, 0xbe, 0x4a, 0x33, 0xb9, 0x3e, 0x55, 0x99, 0x40, 0x8e, 0x61, 0xe9, 0x88, 0x85, 0x78, - 0x16, 0x5c, 0xef, 0xfc, 0x95, 0x3b, 0x0e, 0x79, 0x22, 0xeb, 0x7b, 0xff, 0x37, 0x83, 0x50, 0x74, - 0x3a, 0x4c, 0x62, 0x22, 0xf9, 0x39, 0x54, 0x8e, 0x58, 0xf8, 0xdc, 0xf5, 0x06, 0xfc, 0x54, 0xe3, - 0x39, 0x8d, 0x30, 0x50, 0xa8, 0x4f, 0x8e, 0x4c, 0x09, 0x85, 0x95, 0x97, 0x8c, 0x8d, 0xe2, 0x3c, - 0xe3, 0x7c, 0x71, 0xce, 0xb7, 0xc4, 0xfc, 0x94, 0x5a, 0x47, 0x4a, 0x4f, 0xc7, 0x45, 0xa1, 0x90, - 0xb2, 0x2b, 0x3f, 0x64, 0x3c, 0x2f, 0xa0, 0x2e, 0x4a, 0xd7, 0x25, 0x16, 0xa5, 0x2b, 0xc9, 0x2b, - 0x58, 0xe6, 0x07, 0xd6, 0x09, 0x99, 0x3c, 0xec, 0x56, 0x9d, 0x63, 0x7d, 0x20, 0xb0, 0x12, 0x4a, - 0x1d, 0x2c, 0x39, 0x95, 0x74, 0xa0, 0xd1, 0x71, 0x3c, 0xcf, 0x0f, 0x3b, 0xfe, 0xd5, 0x95, 0x1b, - 0x5a, 0x0d, 0x0e, 0xb5, 0x21, 0xa0, 0x54, 0x8d, 0x8e, 0xa3, 0x4d, 0x42, 0x90, 0x23, 0x16, 0xee, - 0xf7, 0x43, 0xf7, 0x9a, 0xf5, 0xa6, 0x9e, 0xd5, 0x54, 0x41, 0x54, 0x4d, 0x02, 0x44, 0x55, 0x61, - 0xda, 0xbb, 0x2c, 0xa4, 0xc8, 0x1d, 0x41, 0x18, 0xad, 0x6c, 0x49, 0x4d, 0x7b, 0x4a, 0x9d, 0x48, - 0x7b, 0x4a, 0x8f, 0x98, 0x1d, 0xc7, 0x4b, 0x60, 0x2e, 0xab, 0x98, 0x29, 0x75, 0x02, 0x33, 0xa5, - 0x27, 0x6f, 0x80, 0x50, 0x76, 0xe5, 0xb8, 0x5e, 0x6f, 0xea, 0x1d, 0x7b, 0x11, 0xa8, 0xc9, 0x41, - 0x1f, 0x08, 0xd0, 0xb4, 0x5e, 0x47, 0xcd, 0x00, 0x20, 0xbf, 0x86, 0x7a, 0xe7, 0x82, 0xf5, 0x2f, - 0x4f, 0x83, 0xd1, 0x85, 0xe3, 0x59, 0x2b, 0x1c, 0xcf, 0x92, 0x41, 0xc6, 0x0a, 0x1d, 0x48, 0x9d, - 0x82, 0x07, 0x83, 0xb2, 0xf1, 0xe4, 0x8a, 0x1d, 0x7b, 0xd7, 0xb8, 0xcb, 0x9d, 0xd7, 0x16, 0x51, - 0x0f, 0x46, 0x42, 0x99, 0x38, 0x18, 0x09, 0x2d, 0x71, 0x60, 0x7d, 0xff, 0xcc, 0x0f, 0x42, 0x71, - 0xf2, 0x0e, 0x98, 0x33, 0x40, 0x10, 0xdc, 0xdd, 0x35, 0x0e, 0xfa, 0x13, 0x01, 0x9a, 0x6d, 0xa3, - 0x63, 0xe7, 0x00, 0xd9, 0xdf, 0x02, 0x54, 0x29, 0x1b, 0x8f, 0x7c, 0x6f, 0xcc, 0xe6, 0xd0, 0x61, - 0xcc, 0x6c, 0x85, 0x3b, 0x98, 0x6d, 0x0d, 0x16, 0x5f, 0x04, 0x81, 0x1f, 0x70, 0xfa, 0x6b, 0x50, - 0x31, 0x20, 0x1f, 0x43, 0xe5, 0x35, 0xbb, 0xe1, 0x6f, 0x71, 0x29, 0x93, 0x48, 0x69, 0xa4, 0x27, - 0x3f, 0x95, 0xd4, 0x28, 0xb8, 0x8e, 0xa8, 0xd4, 0x28, 0xc2, 0xd4, 0xb8, 0x71, 0x6f, 0xc6, 0x8d, - 0x65, 0x95, 0x5d, 0x22, 0x6e, 0xd4, 0x66, 0x44, 0xe4, 0xf8, 0x4c, 0x23, 0xc7, 0x8a, 0xba, 0xcb, - 0x2a, 0x39, 0x6a, 0x73, 0x55, 0x76, 0xfc, 0x3a, 0xc5, 0x8e, 0x55, 0x75, 0x8f, 0x93, 0xec, 0xa8, - 0xe1, 0x24, 0xe9, 0xf1, 0x49, 0x4c, 0x8f, 0x82, 0xde, 0xee, 0x25, 0xe8, 0x51, 0x9b, 0x3d, 0xe3, - 0xc7, 0x6e, 0x16, 0x3f, 0x82, 0x7a, 0xfe, 0x33, 0xf8, 0x51, 0x83, 0xca, 0x20, 0xc8, 0xaf, 0x53, - 0x04, 0xa9, 0x91, 0x5a, 0x92, 0x20, 0xf5, 0x75, 0x25, 0x18, 0xf2, 0x24, 0xcd, 0x90, 0x82, 0xd6, - 0x3e, 0xcc, 0x61, 0x48, 0x0d, 0x2d, 0x45, 0x91, 0x07, 0x09, 0x8a, 0x14, 0xec, 0xb6, 0x99, 0x45, - 0x91, 0x1a, 0x90, 0xce, 0x91, 0x07, 0x09, 0x8e, 0x5c, 0x52, 0x51, 0x74, 0x8e, 0xd4, 0x51, 0x34, - 0x92, 0xec, 0x66, 0x91, 0xe4, 0xb2, 0x9a, 0xfb, 0x0c, 0x92, 0xd4, 0x73, 0x9f, 0x66, 0xc9, 0x6e, - 0x16, 0x4b, 0x6a, 0x84, 0x96, 0xc1, 0x92, 0x3a, 0x68, 0x9a, 0x26, 0xdf, 0x66, 0xd2, 0xa4, 0xa0, - 0xb5, 0x56, 0x3e, 0x4d, 0x6a, 0xb0, 0x59, 0x3c, 0xb9, 0xaf, 0xf3, 0x24, 0xd1, 0xee, 0x2b, 0x95, - 0x27, 0x35, 0x24, 0x8d, 0x28, 0x4f, 0xd2, 0x44, 0xb9, 0xaa, 0x9e, 0x8f, 0x14, 0x51, 0xea, 0xe7, - 0x23, 0xc9, 0x94, 0x67, 0x73, 0x98, 0x72, 0xfb, 0x6e, 0xa6, 0xd4, 0xc0, 0xf3, 0xa8, 0xf2, 0x8f, - 0x86, 0xe8, 0x57, 0xa2, 0xe2, 0x11, 0x0b, 0xe2, 0xa9, 0x27, 0x99, 0xb2, 0x41, 0xc5, 0x60, 0x4e, - 0x41, 0x4c, 0xa0, 0x44, 0xfd, 0x9b, 0xb1, 0x55, 0x6c, 0x15, 0xdb, 0x0d, 0xca, 0x9f, 0xc9, 0x23, - 0xa8, 0xc8, 0x16, 0x28, 0x5d, 0x0e, 0x4a, 0x45, 0xf4, 0xfa, 0xcb, 0xa1, 0xfd, 0x14, 0x1a, 0xea, - 0x3b, 0x48, 0x76, 0xa0, 0x8c, 0x19, 0x19, 0x86, 0x3c, 0x96, 0x7a, 0x44, 0xcd, 0x42, 0x16, 0xb1, - 0x9f, 0x18, 0xd9, 0x5f, 0xc2, 0x4a, 0xaa, 0x04, 0xcc, 0x59, 0x8b, 0x09, 0x45, 0xea, 0xdf, 0xf0, - 0x55, 0x34, 0x28, 0x3e, 0xda, 0x0e, 0x90, 0x34, 0x45, 0xca, 0x62, 0x7e, 0x22, 0x5a, 0x83, 0x45, - 0x2a, 0x06, 0xe4, 0x09, 0xd4, 0x55, 0x8e, 0x2c, 0xb4, 0x8a, 0xed, 0xfa, 0x5e, 0x33, 0xee, 0xbd, - 0x7a, 0x53, 0x2f, 0x3a, 0x19, 0x8a, 0x9d, 0xfd, 0x0c, 0xee, 0x65, 0xd6, 0x97, 0xe4, 0x23, 0x28, - 0xe2, 0x86, 0x8a, 0x15, 0x66, 0xe2, 0xa0, 0xde, 0x3e, 0x85, 0xf5, 0x6c, 0x06, 0x4e, 0x06, 0x64, - 0xfc, 0xc0, 0x80, 0xfa, 0x50, 0x91, 0xda, 0xfc, 0x2d, 0xef, 0x04, 0xcc, 0x09, 0xd9, 0xe0, 0xd4, - 0x8b, 0xb6, 0x7c, 0x26, 0x20, 0xdb, 0xd0, 0xc4, 0xe9, 0x2c, 0xd8, 0x1f, 0x0c, 0x02, 0x36, 0x1e, - 0xf3, 0x8b, 0xb1, 0x46, 0x75, 0xa1, 0xfd, 0x67, 0x03, 0x9a, 0x5a, 0x41, 0x9f, 0xe3, 0xeb, 0x33, - 0xa8, 0x0a, 0x32, 0xeb, 0x75, 0x65, 0x4b, 0x72, 0x57, 0xdf, 0x37, 0xb3, 0x25, 0x9f, 0x43, 0xed, - 0x64, 0x12, 0x3a, 0xe2, 0x98, 0x15, 0xf9, 0xca, 0x65, 0x1b, 0xf1, 0x62, 0x1a, 0x06, 0x4e, 0xa4, - 0x8b, 0xfa, 0x99, 0x99, 0xad, 0x6d, 0xc2, 0x92, 0x7e, 0x99, 0xda, 0xdf, 0x1a, 0xfc, 0xfe, 0x53, - 0x6a, 0x6e, 0xfd, 0xd0, 0x1b, 0xc9, 0x43, 0x3f, 0xeb, 0x1c, 0x0b, 0x6a, 0xe7, 0x38, 0xeb, 0xf5, - 0x8a, 0x79, 0xbd, 0x5e, 0xe9, 0xc7, 0xf5, 0x7a, 0x8b, 0xe9, 0x5e, 0xef, 0x10, 0x96, 0x13, 0x17, - 0xe9, 0xff, 0xd4, 0xd4, 0xd9, 0x7f, 0x33, 0xc0, 0xca, 0x6b, 0x38, 0xe6, 0x2c, 0x7e, 0x1b, 0xca, - 0xdd, 0xd0, 0x09, 0x27, 0x63, 0xbd, 0x6a, 0x12, 0x32, 0x2a, 0x75, 0x64, 0x1d, 0xca, 0x7c, 0x7f, - 0x23, 0x66, 0x90, 0x23, 0xf2, 0x04, 0x60, 0xe6, 0x13, 0xe9, 0xa1, 0x98, 0x1f, 0xae, 0x62, 0x68, - 0xff, 0x06, 0x36, 0x72, 0xef, 0x7f, 0xb2, 0x04, 0x85, 0xd3, 0x97, 0x3c, 0xd0, 0x2a, 0x2d, 0x9c, - 0xbe, 0xfc, 0x61, 0x11, 0xda, 0x5f, 0x80, 0x95, 0x57, 0xfb, 0xdf, 0x9d, 0x01, 0xfb, 0x13, 0xd8, - 0xc8, 0xbd, 0x10, 0x93, 0xc1, 0xa0, 0x9b, 0xbc, 0x76, 0x60, 0xbe, 0x9b, 0xdc, 0x2b, 0x32, 0xe5, - 0xe6, 0x17, 0xb0, 0x91, 0xdb, 0x20, 0xcc, 0xf1, 0xf3, 0x14, 0x36, 0xf3, 0x2f, 0x4d, 0x51, 0x42, - 0x4b, 0xad, 0xa4, 0xc3, 0x58, 0x60, 0x3f, 0x81, 0x7b, 0x99, 0x6d, 0xe6, 0x1c, 0x97, 0x6d, 0x58, - 0xcf, 0x2e, 0xbe, 0x52, 0xeb, 0xfa, 0x0c, 0xd6, 0xb3, 0x7b, 0xcf, 0x39, 0x1e, 0x3e, 0x86, 0xfb, - 0x39, 0x15, 0x59, 0xca, 0xc5, 0x21, 0x7c, 0x78, 0x67, 0xc3, 0x81, 0x3c, 0x1d, 0xce, 0xe1, 0xe9, - 0x70, 0xea, 0xd9, 0x3f, 0x83, 0xad, 0xbb, 0xaf, 0xe3, 0x94, 0x67, 0x0a, 0xab, 0x19, 0xdd, 0x30, - 0xf9, 0x12, 0x9a, 0xa2, 0xa8, 0xc0, 0x6b, 0x29, 0x26, 0x76, 0xf9, 0x9a, 0xcc, 0x54, 0xd2, 0xb7, - 0x6e, 0x6b, 0xff, 0x12, 0xd6, 0xb2, 0xca, 0x47, 0x64, 0x6d, 0x21, 0xc1, 0x6b, 0x40, 0xec, 0x25, - 0xbe, 0x97, 0xba, 0xd0, 0x7e, 0x0c, 0xab, 0x19, 0xad, 0xf5, 0x9c, 0x5c, 0x3f, 0x83, 0xb5, 0xac, - 0x5a, 0x33, 0xfe, 0x24, 0x66, 0xa8, 0x9f, 0xc4, 0x4c, 0x71, 0xeb, 0x15, 0xb8, 0x7b, 0x7e, 0xc1, - 0x1d, 0x00, 0x49, 0x37, 0xa3, 0x73, 0x58, 0x68, 0x86, 0x62, 0x44, 0x28, 0x9f, 0xc2, 0x6a, 0x46, - 0xa9, 0x86, 0x44, 0x24, 0xab, 0x3a, 0x11, 0x85, 0x1c, 0xd9, 0x9f, 0x43, 0x6d, 0x96, 0x38, 0x62, - 0x41, 0x25, 0x2a, 0x26, 0x85, 0xa7, 0x68, 0x98, 0x11, 0xed, 0x1f, 0x8a, 0x51, 0x6d, 0x42, 0x1e, - 0x41, 0x15, 0x0f, 0x2f, 0xbf, 0x26, 0x8d, 0xbb, 0x98, 0x77, 0x66, 0x86, 0x14, 0xff, 0x95, 0x33, - 0xee, 0xf8, 0xde, 0xbb, 0xa1, 0xdb, 0x0f, 0x79, 0xfc, 0x55, 0xaa, 0x8a, 0x70, 0xa3, 0xbe, 0x72, - 0xc6, 0xdf, 0x04, 0xec, 0x5a, 0xb6, 0x06, 0x45, 0x6e, 0xa3, 0x0b, 0xc9, 0x17, 0x50, 0x9b, 0xdd, - 0x8d, 0xb2, 0xca, 0xba, 0xeb, 0xde, 0x8c, 0x8d, 0x7f, 0xc4, 0x47, 0xdf, 0x16, 0xd4, 0xa3, 0xa8, - 0x5e, 0xb2, 0x5b, 0xde, 0x8f, 0x36, 0xa8, 0x2a, 0x52, 0x2d, 0x30, 0x4b, 0x15, 0xdd, 0x02, 0x33, - 0xbb, 0x05, 0x80, 0x51, 0x8b, 0xda, 0x80, 0xb7, 0x95, 0x0d, 0xaa, 0x48, 0x30, 0xf3, 0xe2, 0x49, - 0x7c, 0xf5, 0x6d, 0xd2, 0x68, 0x88, 0x33, 0x5f, 0xb3, 0x1b, 0x4c, 0xdc, 0xfe, 0x40, 0xb4, 0x82, - 0x55, 0xaa, 0x48, 0xec, 0x2e, 0x34, 0xb5, 0x9b, 0x1e, 0xb7, 0xea, 0x92, 0xdd, 0xca, 0xea, 0x02, - 0x1f, 0xb1, 0x38, 0x1d, 0x5f, 0xba, 0x23, 0x99, 0x65, 0xfe, 0x8c, 0xc7, 0x2a, 0x60, 0xa3, 0xa1, - 0xd3, 0x67, 0x3d, 0x5f, 0xb6, 0xf4, 0xb1, 0x00, 0xe9, 0x26, 0xfb, 0x8b, 0xc6, 0x9c, 0x57, 0x60, - 0x03, 0xee, 0xe7, 0x14, 0xf8, 0x3b, 0xff, 0xaf, 0xfd, 0x05, 0x20, 0x15, 0x5e, 0x82, 0x9a, 0x0b, - 0xa4, 0x06, 0x8b, 0x14, 0xf3, 0x6c, 0x1a, 0x3b, 0x1f, 0x89, 0x73, 0xc4, 0xbf, 0xed, 0x37, 0xa1, - 0xf6, 0x62, 0xda, 0x1f, 0x4e, 0xc6, 0xee, 0x35, 0x33, 0x17, 0x08, 0x40, 0x19, 0xcb, 0x03, 0x36, - 0x30, 0x8d, 0x9d, 0x6d, 0x80, 0xf8, 0x13, 0x3f, 0xa9, 0x42, 0x09, 0x47, 0xe6, 0x02, 0x69, 0x40, - 0xf5, 0xd0, 0x19, 0x87, 0x87, 0x8e, 0x3b, 0x34, 0x8d, 0x9d, 0x07, 0x71, 0xc1, 0x81, 0x36, 0xaf, - 0x7d, 0x8f, 0x09, 0x6f, 0xcf, 0x6f, 0xd1, 0xb1, 0xb1, 0xf3, 0xcf, 0x42, 0xf4, 0xdd, 0x03, 0xf5, - 0xe8, 0x58, 0xf8, 0x11, 0x55, 0x91, 0x69, 0x90, 0x25, 0xf5, 0x73, 0x82, 0x59, 0x20, 0x24, 0xf9, - 0x79, 0xc0, 0x2c, 0xa2, 0x4c, 0x67, 0x70, 0xb3, 0x44, 0xea, 0xb3, 0xd6, 0xdf, 0x5c, 0x24, 0xf7, - 0x32, 0x1a, 0x7a, 0xb3, 0x4c, 0x96, 0xa1, 0x2e, 0xff, 0x3f, 0xf0, 0x49, 0x15, 0xb2, 0x02, 0x4d, - 0x29, 0x90, 0xfe, 0xab, 0x64, 0x35, 0xd5, 0x6a, 0x9b, 0x35, 0x62, 0xea, 0x0d, 0xb3, 0x09, 0x28, - 0x51, 0x69, 0xc7, 0xac, 0xa3, 0xcf, 0xd4, 0xc5, 0x6c, 0x36, 0xc8, 0x7a, 0x56, 0xd7, 0x68, 0x36, - 0xd1, 0x3c, 0x75, 0xc1, 0x9a, 0x4b, 0x18, 0xa2, 0x42, 0x24, 0xe6, 0x32, 0xc6, 0x93, 0xd8, 0x5c, - 0xd3, 0x24, 0x9b, 0x79, 0x0d, 0x9a, 0xb9, 0xb2, 0xc3, 0xa2, 0x02, 0x44, 0x44, 0xc4, 0x81, 0x71, - 0xb9, 0x2f, 0x3c, 0xcc, 0x84, 0xb9, 0x80, 0x11, 0x29, 0x62, 0x99, 0x59, 0xd3, 0x50, 0xcc, 0xdf, - 0xf0, 0xe4, 0x77, 0x27, 0xfd, 0xbe, 0x59, 0x50, 0xc4, 0x71, 0xbc, 0x66, 0xf1, 0x79, 0xe7, 0xfb, - 0x7f, 0x6f, 0x19, 0xdf, 0xbd, 0xdf, 0x32, 0xbe, 0x7f, 0xbf, 0x65, 0xfc, 0xeb, 0xfd, 0xd6, 0xc2, - 0x5f, 0xff, 0xb3, 0x65, 0xfc, 0x4e, 0xfd, 0x8b, 0x75, 0xe5, 0x84, 0x81, 0x3b, 0xf5, 0x79, 0x01, - 0x19, 0x0d, 0x3c, 0xb6, 0x3b, 0xba, 0x3c, 0xdf, 0x1d, 0x9d, 0xed, 0x62, 0xc0, 0x67, 0x65, 0xfe, - 0xef, 0xea, 0xf1, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x26, 0x07, 0x19, 0xe1, 0x0f, 0x1b, 0x00, - 0x00, + // 2130 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x59, 0xcd, 0x73, 0xdb, 0xc6, + 0x15, 0x17, 0x48, 0x8a, 0x1f, 0x8f, 0xa4, 0x04, 0xad, 0x64, 0x19, 0x52, 0x13, 0x99, 0x45, 0x95, + 0x19, 0x46, 0x69, 0xac, 0x5a, 0xae, 0x93, 0xd4, 0x69, 0x3d, 0x95, 0x29, 0x5b, 0x51, 0x6c, 0x59, + 0x99, 0x25, 0xed, 0xce, 0xf4, 0x06, 0x91, 0x6b, 0x09, 0x23, 0x0a, 0x60, 0x41, 0x50, 0xa2, 0x0e, + 0x3d, 0x76, 0xa6, 0x33, 0xbd, 0xf7, 0xd8, 0xe9, 0x31, 0x97, 0xfe, 0x1f, 0x39, 0x66, 0xa6, 0xf7, + 0x7e, 0xb8, 0xff, 0x48, 0xe7, 0xed, 0x2e, 0x88, 0x5d, 0x7c, 0x88, 0x71, 0x6e, 0xd8, 0xf7, 0xde, + 0xfe, 0xde, 0xee, 0xdb, 0x87, 0xdf, 0xbe, 0x07, 0x00, 0x0c, 0xfd, 0xfe, 0xc5, 0xfd, 0x51, 0xe0, + 0x87, 0x3e, 0x29, 0xe1, 0xf3, 0xe6, 0xa7, 0x67, 0x6e, 0x78, 0x3e, 0x39, 0xbd, 0xdf, 0xf7, 0x2f, + 0x77, 0xcf, 0xfc, 0x33, 0x7f, 0x97, 0x2b, 0x4f, 0x27, 0x6f, 0xf9, 0x88, 0x0f, 0xf8, 0x93, 0x98, + 0xb4, 0xb9, 0x1c, 0xba, 0x97, 0x6c, 0x1c, 0x3a, 0x97, 0x23, 0x21, 0xb0, 0xff, 0x56, 0x84, 0xfa, + 0x4b, 0xbf, 0x7f, 0x71, 0x32, 0x0a, 0x5d, 0xdf, 0x1b, 0x93, 0x87, 0x50, 0x3f, 0x0c, 0x1c, 0x6f, + 0x32, 0x74, 0x02, 0x37, 0xbc, 0xb1, 0x8c, 0x96, 0xd1, 0x5e, 0xda, 0x5b, 0xb9, 0xcf, 0xfd, 0x2a, + 0x0a, 0xaa, 0x5a, 0x11, 0x1b, 0x4a, 0xc7, 0xfe, 0x80, 0x59, 0x05, 0x6e, 0xbd, 0x24, 0xac, 0x11, + 0x15, 0xa5, 0x94, 0xeb, 0x48, 0x1b, 0xca, 0xdf, 0xf8, 0x43, 0xb7, 0x7f, 0x63, 0x15, 0xb9, 0x95, + 0x29, 0xac, 0x7e, 0xe7, 0xb8, 0xa1, 0x90, 0x53, 0xa9, 0x27, 0x1f, 0x40, 0xed, 0xb9, 0x1f, 0x5c, + 0x3b, 0xc1, 0xa0, 0xe7, 0x5b, 0xa5, 0x96, 0xd1, 0xae, 0xd1, 0x58, 0x40, 0xda, 0xb0, 0xdc, 0x73, + 0x4e, 0x87, 0xec, 0x80, 0xbd, 0xed, 0x9c, 0x3b, 0xde, 0x19, 0x1b, 0x58, 0x8b, 0x2d, 0xa3, 0x5d, + 0xa5, 0x49, 0x31, 0xe2, 0x50, 0x16, 0x06, 0x37, 0xe8, 0xc2, 0x2a, 0xb7, 0x8c, 0x76, 0x91, 0xc6, + 0x02, 0xb2, 0x06, 0x8b, 0x87, 0x81, 0x3f, 0x19, 0x59, 0x95, 0x96, 0xd1, 0x6e, 0x52, 0x31, 0x20, + 0x3b, 0x50, 0xed, 0x9e, 0x3b, 0xc1, 0xc0, 0xf5, 0xce, 0xac, 0xaa, 0xba, 0x9b, 0x48, 0x4a, 0x67, + 0x7a, 0xf2, 0x18, 0xa0, 0xeb, 0x39, 0xa3, 0xee, 0xb9, 0x1f, 0xf6, 0xc6, 0x56, 0xad, 0x65, 0xb4, + 0xeb, 0x7b, 0x6b, 0xf7, 0xe3, 0x00, 0xf7, 0xa2, 0xa7, 0xa7, 0xa5, 0xef, 0xfe, 0x75, 0x6f, 0x81, + 0x2a, 0xd6, 0xb8, 0x0b, 0x8c, 0x0f, 0xae, 0x04, 0xcd, 0xfc, 0x49, 0x68, 0x01, 0x5f, 0x61, 0x52, + 0x6c, 0xff, 0xd3, 0x80, 0x1a, 0xca, 0xf8, 0xee, 0x70, 0xd5, 0xfc, 0x81, 0x1f, 0x4c, 0x89, 0x8a, + 0x01, 0xee, 0xb4, 0xcb, 0x82, 0x2b, 0xb7, 0xcf, 0x8e, 0x0e, 0xf8, 0x21, 0xd4, 0x68, 0x2c, 0x20, + 0x16, 0x54, 0xde, 0xb0, 0x60, 0xec, 0xfa, 0x1e, 0x0f, 0x7d, 0x89, 0x46, 0x43, 0x44, 0x7b, 0xe3, + 0x0c, 0xdd, 0x01, 0x8f, 0x72, 0x95, 0x8a, 0x41, 0x1c, 0x99, 0xc5, 0xbc, 0xc8, 0x94, 0xe7, 0x44, + 0xa6, 0x05, 0xf5, 0x93, 0xc0, 0x3d, 0x73, 0x3d, 0xb1, 0xd6, 0x0a, 0xf7, 0xaa, 0x8a, 0xec, 0xbf, + 0x00, 0x54, 0x28, 0xfb, 0xc3, 0x84, 0x8d, 0x43, 0x71, 0x4e, 0xfc, 0xf1, 0xe8, 0x40, 0xee, 0x2b, + 0x16, 0x90, 0x87, 0xca, 0xf6, 0xf9, 0xde, 0xea, 0x7b, 0xcb, 0x71, 0x82, 0x71, 0xb1, 0x8c, 0xaf, + 0x12, 0xa6, 0x6d, 0x28, 0x1f, 0xb3, 0xf0, 0xdc, 0x1f, 0xc8, 0x64, 0x6b, 0x88, 0x19, 0x42, 0x46, + 0xa5, 0x8e, 0x7c, 0x02, 0x25, 0x9c, 0xc2, 0x77, 0x5f, 0x8f, 0x92, 0x1c, 0x25, 0xd2, 0xbb, 0xc4, + 0xe5, 0x46, 0xe4, 0x01, 0x94, 0x5f, 0x7b, 0x68, 0xc1, 0xc3, 0x52, 0xdf, 0x5b, 0x15, 0xe6, 0x42, + 0xa6, 0x4f, 0x90, 0x86, 0xe4, 0x37, 0x00, 0x87, 0x2c, 0xec, 0x4d, 0x3d, 0xee, 0xa5, 0xcc, 0xa7, + 0xdd, 0x95, 0xaf, 0xd2, 0x4c, 0xae, 0x4f, 0x55, 0x26, 0x90, 0x23, 0x58, 0x3a, 0x64, 0x21, 0xe6, + 0x82, 0xeb, 0x9d, 0xbd, 0x74, 0xc7, 0x21, 0x0f, 0x64, 0x7d, 0xef, 0x27, 0x33, 0x08, 0x45, 0xa7, + 0xc3, 0x24, 0x26, 0x92, 0x5f, 0x42, 0xe5, 0x90, 0x85, 0x4f, 0x5d, 0x6f, 0xc0, 0xb3, 0x1a, 0xf3, + 0x34, 0xc2, 0x40, 0xa1, 0x3e, 0x39, 0x32, 0x25, 0x14, 0x56, 0x5e, 0x30, 0x36, 0x8a, 0xe3, 0x8c, + 0xf3, 0x45, 0x9e, 0x6f, 0x89, 0xf9, 0x29, 0xb5, 0x8e, 0x94, 0x9e, 0x8e, 0x9b, 0x42, 0x21, 0x65, + 0x97, 0x7e, 0xc8, 0x78, 0x5c, 0x40, 0xdd, 0x94, 0xae, 0x4b, 0x6c, 0x4a, 0x57, 0x92, 0x97, 0xb0, + 0xcc, 0x13, 0xd6, 0x09, 0x99, 0x4c, 0x76, 0xab, 0xce, 0xb1, 0x3e, 0x10, 0x58, 0x09, 0xa5, 0x0e, + 0x96, 0x9c, 0x4a, 0x3a, 0xd0, 0xe8, 0x38, 0x9e, 0xe7, 0x87, 0x1d, 0xff, 0xf2, 0xd2, 0x0d, 0xad, + 0x06, 0x87, 0xda, 0x10, 0x50, 0xaa, 0x46, 0xc7, 0xd1, 0x26, 0x21, 0xc8, 0x21, 0x0b, 0xf7, 0xfb, + 0xa1, 0x7b, 0xc5, 0x7a, 0x53, 0xcf, 0x6a, 0xaa, 0x20, 0xaa, 0x26, 0x01, 0xa2, 0xaa, 0x30, 0xec, + 0x5d, 0x16, 0x52, 0xe4, 0x8e, 0x20, 0x8c, 0x76, 0xb6, 0xa4, 0x86, 0x3d, 0xa5, 0x4e, 0x84, 0x3d, + 0xa5, 0x47, 0xcc, 0x8e, 0xe3, 0x25, 0x30, 0x97, 0x55, 0xcc, 0x94, 0x3a, 0x81, 0x99, 0xd2, 0x93, + 0xd7, 0x40, 0x28, 0xbb, 0x74, 0x5c, 0xaf, 0x37, 0xf5, 0x8e, 0xbc, 0x08, 0xd4, 0xe4, 0xa0, 0xf7, + 0x04, 0x68, 0x5a, 0xaf, 0xa3, 0x66, 0x00, 0x90, 0xdf, 0x42, 0xbd, 0x73, 0xce, 0xfa, 0x17, 0x27, + 0xc1, 0xe8, 0xdc, 0xf1, 0xac, 0x15, 0x8e, 0x67, 0xc9, 0x45, 0xc6, 0x0a, 0x1d, 0x48, 0x9d, 0x82, + 0x89, 0x41, 0xd9, 0x78, 0x72, 0xc9, 0x8e, 0xbc, 0x2b, 0x3c, 0xe5, 0xce, 0x2b, 0x8b, 0xa8, 0x89, + 0x91, 0x50, 0x26, 0x12, 0x23, 0xa1, 0x25, 0x0e, 0xac, 0xef, 0x9f, 0xfa, 0x41, 0x28, 0x32, 0xef, + 0x80, 0x39, 0x03, 0x04, 0xc1, 0xd3, 0x5d, 0xe3, 0xa0, 0x3f, 0x13, 0xa0, 0xd9, 0x36, 0x3a, 0x76, + 0x0e, 0x90, 0xfd, 0x2d, 0x40, 0x95, 0xb2, 0xf1, 0xc8, 0xf7, 0xc6, 0x6c, 0x0e, 0x1d, 0xc6, 0xcc, + 0x56, 0xb8, 0x85, 0xd9, 0xd6, 0x60, 0xf1, 0x59, 0x10, 0xf8, 0x01, 0xa7, 0xbf, 0x06, 0x15, 0x03, + 0xf2, 0x31, 0x54, 0x5e, 0xb1, 0x6b, 0xfe, 0x16, 0x97, 0x32, 0x89, 0x94, 0x46, 0x7a, 0xf2, 0x73, + 0x49, 0x8d, 0x82, 0xeb, 0x88, 0x4a, 0x8d, 0x62, 0x99, 0x1a, 0x37, 0xee, 0xcd, 0xb8, 0xb1, 0xac, + 0xb2, 0x4b, 0xc4, 0x8d, 0xda, 0x8c, 0x88, 0x1c, 0x9f, 0x68, 0xe4, 0x58, 0x51, 0x4f, 0x59, 0x25, + 0x47, 0x6d, 0xae, 0xca, 0x8e, 0x5f, 0xa7, 0xd8, 0xb1, 0xaa, 0x9e, 0x71, 0x92, 0x1d, 0x35, 0x9c, + 0x24, 0x3d, 0x3e, 0x8a, 0xe9, 0x51, 0xd0, 0xdb, 0x9d, 0x04, 0x3d, 0x6a, 0xb3, 0x67, 0xfc, 0xd8, + 0xcd, 0xe2, 0x47, 0x50, 0xf3, 0x3f, 0x83, 0x1f, 0x35, 0xa8, 0x0c, 0x82, 0xfc, 0x3a, 0x45, 0x90, + 0x1a, 0xa9, 0x25, 0x09, 0x52, 0xdf, 0x57, 0x82, 0x21, 0x8f, 0xd3, 0x0c, 0x29, 0x68, 0xed, 0xc3, + 0x1c, 0x86, 0xd4, 0xd0, 0x52, 0x14, 0x79, 0x90, 0xa0, 0x48, 0xc1, 0x6e, 0x9b, 0x59, 0x14, 0xa9, + 0x01, 0xe9, 0x1c, 0x79, 0x90, 0xe0, 0xc8, 0x25, 0x15, 0x45, 0xe7, 0x48, 0x1d, 0x45, 0x23, 0xc9, + 0x6e, 0x16, 0x49, 0x2e, 0xab, 0xb1, 0xcf, 0x20, 0x49, 0x3d, 0xf6, 0x69, 0x96, 0xec, 0x66, 0xb1, + 0xa4, 0x46, 0x68, 0x19, 0x2c, 0xa9, 0x83, 0xa6, 0x69, 0xf2, 0x4d, 0x26, 0x4d, 0x0a, 0x5a, 0x6b, + 0xe5, 0xd3, 0xa4, 0x06, 0x9b, 0xc5, 0x93, 0xfb, 0x3a, 0x4f, 0x12, 0xed, 0xbe, 0x52, 0x79, 0x52, + 0x43, 0xd2, 0x88, 0xf2, 0x38, 0x4d, 0x94, 0xab, 0x6a, 0x7e, 0xa4, 0x88, 0x52, 0xcf, 0x8f, 0x24, + 0x53, 0x9e, 0xce, 0x61, 0xca, 0xed, 0xdb, 0x99, 0x52, 0x03, 0xcf, 0xa3, 0xca, 0x3f, 0x1b, 0xa2, + 0x5f, 0x89, 0x8a, 0x47, 0x2c, 0x88, 0xa7, 0x9e, 0x64, 0xca, 0x06, 0x15, 0x83, 0x39, 0x05, 0x31, + 0x81, 0x12, 0xf5, 0xaf, 0xc7, 0x56, 0xb1, 0x55, 0x6c, 0x37, 0x28, 0x7f, 0x26, 0x0f, 0xa0, 0x22, + 0x5b, 0xa0, 0x74, 0x39, 0x28, 0x15, 0xd1, 0xeb, 0x2f, 0x87, 0xf6, 0x63, 0x68, 0xa8, 0xef, 0x20, + 0xd9, 0x81, 0x32, 0x46, 0x64, 0x18, 0xf2, 0xb5, 0xd4, 0x23, 0x6a, 0x16, 0xb2, 0x88, 0xfd, 0xc4, + 0xc8, 0xfe, 0x12, 0x56, 0x52, 0x25, 0x60, 0xce, 0x5e, 0x4c, 0x28, 0x52, 0xff, 0x9a, 0xef, 0xa2, + 0x41, 0xf1, 0xd1, 0x76, 0x80, 0xa4, 0x29, 0x52, 0x16, 0xf3, 0x13, 0xd1, 0x1a, 0x2c, 0x52, 0x31, + 0x20, 0x8f, 0xa0, 0xae, 0x72, 0x64, 0xa1, 0x55, 0x6c, 0xd7, 0xf7, 0x9a, 0x71, 0xef, 0xd5, 0x9b, + 0x7a, 0x51, 0x66, 0x28, 0x76, 0xf6, 0x13, 0xb8, 0x93, 0x59, 0x5f, 0x92, 0x8f, 0xa0, 0x88, 0x07, + 0x2a, 0x76, 0x98, 0x89, 0x83, 0x7a, 0xfb, 0x04, 0xd6, 0xb3, 0x19, 0x38, 0xb9, 0x20, 0xe3, 0x07, + 0x2e, 0xa8, 0x0f, 0x15, 0xa9, 0xcd, 0x3f, 0xf2, 0x4e, 0xc0, 0x9c, 0x90, 0x0d, 0x4e, 0xbc, 0xe8, + 0xc8, 0x67, 0x02, 0xb2, 0x0d, 0x4d, 0x9c, 0xce, 0x82, 0xfd, 0xc1, 0x20, 0x60, 0xe3, 0x31, 0xbf, + 0x18, 0x6b, 0x54, 0x17, 0xda, 0x7f, 0x35, 0xa0, 0xa9, 0x15, 0xf4, 0x39, 0xbe, 0x3e, 0x83, 0xaa, + 0x20, 0xb3, 0x5e, 0x57, 0xb6, 0x24, 0xb7, 0xf5, 0x7d, 0x33, 0x5b, 0xf2, 0x39, 0xd4, 0x8e, 0x27, + 0xa1, 0x23, 0xd2, 0xac, 0xc8, 0x77, 0x2e, 0xdb, 0x88, 0x67, 0xd3, 0x30, 0x70, 0x22, 0x5d, 0xd4, + 0xcf, 0xcc, 0x6c, 0x6d, 0x13, 0x96, 0xf4, 0xcb, 0xd4, 0xfe, 0xd6, 0xe0, 0xf7, 0x9f, 0x52, 0x73, + 0xeb, 0x49, 0x6f, 0x24, 0x93, 0x7e, 0xd6, 0x39, 0x16, 0xd4, 0xce, 0x71, 0xd6, 0xeb, 0x15, 0xf3, + 0x7a, 0xbd, 0xd2, 0xfb, 0xf5, 0x7a, 0x8b, 0xe9, 0x5e, 0x2f, 0x80, 0xe5, 0xc4, 0x45, 0xfa, 0xe3, + 0x9a, 0xba, 0x1d, 0x30, 0xf7, 0x87, 0x43, 0xbf, 0xef, 0x84, 0x7e, 0xa0, 0x37, 0xb4, 0x29, 0xb9, + 0xfd, 0x0f, 0x03, 0xac, 0xbc, 0xe6, 0x64, 0x4e, 0xa0, 0xb6, 0xa1, 0xdc, 0x0d, 0x9d, 0x70, 0x32, + 0xd6, 0x2b, 0x2c, 0x21, 0xa3, 0x52, 0x47, 0xd6, 0xa1, 0xcc, 0x73, 0x21, 0x62, 0x11, 0x39, 0x22, + 0x8f, 0x00, 0x66, 0x3e, 0x91, 0x4a, 0x8a, 0xf9, 0x5b, 0x53, 0x0c, 0xed, 0x3f, 0xc2, 0x46, 0x6e, + 0xad, 0x40, 0x96, 0xa0, 0x70, 0xf2, 0x82, 0x2f, 0xb4, 0x4a, 0x0b, 0x27, 0x2f, 0x7e, 0xe0, 0x0a, + 0xdf, 0x27, 0x5c, 0x5f, 0x80, 0x95, 0xd7, 0x53, 0xdc, 0x1e, 0x2d, 0xfb, 0x13, 0xd8, 0xc8, 0xbd, + 0x68, 0x93, 0x0b, 0x47, 0x37, 0x79, 0x6d, 0xc6, 0x7c, 0x37, 0xb9, 0x57, 0x6f, 0xca, 0xcd, 0xaf, + 0x60, 0x23, 0xb7, 0xf1, 0x98, 0xe3, 0xe7, 0x31, 0x6c, 0xe6, 0x5f, 0xc6, 0xa2, 0x34, 0x97, 0x5a, + 0x49, 0xb3, 0xb1, 0xc0, 0x7e, 0x04, 0x77, 0x32, 0xdb, 0xd7, 0x39, 0x2e, 0xdb, 0xb0, 0x9e, 0x5d, + 0xd4, 0xa5, 0xf6, 0xf5, 0x19, 0xac, 0x67, 0xf7, 0xb4, 0x73, 0x3c, 0x7c, 0x0c, 0x77, 0x73, 0x2a, + 0xbd, 0x94, 0x8b, 0xe7, 0xf0, 0xe1, 0xad, 0x8d, 0x0c, 0xf2, 0x7f, 0x38, 0x87, 0xff, 0xc3, 0xa9, + 0x67, 0xff, 0x02, 0xb6, 0x6e, 0xbf, 0xe6, 0x53, 0x9e, 0x29, 0xac, 0x66, 0x74, 0xd9, 0xe4, 0x4b, + 0x68, 0x8a, 0x62, 0x05, 0xaf, 0xbb, 0xf8, 0xc2, 0x90, 0xaf, 0xd4, 0x4c, 0x25, 0x7d, 0xeb, 0xb6, + 0xf6, 0xaf, 0x61, 0x2d, 0xab, 0x2c, 0xc5, 0xdb, 0x40, 0x48, 0xf0, 0x7a, 0x11, 0x67, 0x89, 0xef, + 0xb0, 0x2e, 0xb4, 0x1f, 0xc2, 0x6a, 0x46, 0xcb, 0x3e, 0x27, 0xd6, 0x4f, 0x60, 0x2d, 0xab, 0x86, + 0x8d, 0x3f, 0xb5, 0x19, 0xea, 0xa7, 0x36, 0x53, 0xdc, 0xa6, 0x05, 0xee, 0x9e, 0x5f, 0x9c, 0x07, + 0x40, 0xd2, 0x4d, 0xee, 0x1c, 0xc6, 0x9a, 0xa1, 0x18, 0x11, 0xca, 0xa7, 0xb0, 0x9a, 0x51, 0x02, + 0x22, 0x69, 0xc9, 0x6a, 0x51, 0xac, 0x42, 0x8e, 0xec, 0xcf, 0xa1, 0x36, 0x0b, 0x1c, 0xb1, 0xa0, + 0x12, 0x15, 0xa9, 0xc2, 0x53, 0x34, 0xcc, 0x58, 0xed, 0x9f, 0x8a, 0x51, 0xcd, 0x43, 0x1e, 0x40, + 0x15, 0x93, 0x97, 0x5f, 0xbf, 0xc6, 0x6d, 0x8c, 0x3e, 0x33, 0xc3, 0xab, 0xe3, 0x2b, 0x67, 0xdc, + 0xf1, 0xbd, 0xb7, 0x43, 0xb7, 0x1f, 0xf2, 0xf5, 0x57, 0xa9, 0x2a, 0xc2, 0x83, 0xfa, 0xca, 0x19, + 0x7f, 0x13, 0xb0, 0x2b, 0xd9, 0x72, 0x14, 0xb9, 0x8d, 0x2e, 0x24, 0x5f, 0x40, 0x6d, 0x76, 0xe7, + 0xca, 0xea, 0xed, 0xb6, 0xfb, 0x38, 0x36, 0x7e, 0x8f, 0x8f, 0xc9, 0x2d, 0xa8, 0x47, 0xab, 0x7a, + 0xc1, 0x6e, 0x78, 0x9f, 0xdb, 0xa0, 0xaa, 0x48, 0xb5, 0xc0, 0x28, 0x55, 0x74, 0x0b, 0x8c, 0xec, + 0x16, 0x00, 0xae, 0x5a, 0xd4, 0x1c, 0xbc, 0x5d, 0x6d, 0x50, 0x45, 0x82, 0x91, 0x17, 0x4f, 0xe2, + 0x6b, 0x72, 0x93, 0x46, 0x43, 0x9c, 0xf9, 0x8a, 0x5d, 0x63, 0xe0, 0xf6, 0x07, 0xa2, 0xc5, 0xac, + 0x52, 0x45, 0x62, 0x77, 0xa1, 0xa9, 0x55, 0x10, 0x78, 0x54, 0x17, 0xec, 0x46, 0x56, 0x2d, 0xf8, + 0x88, 0x45, 0xef, 0xf8, 0xc2, 0x1d, 0xc9, 0x28, 0xf3, 0x67, 0x4c, 0xab, 0x80, 0x8d, 0x86, 0x4e, + 0x9f, 0xf5, 0x7c, 0xf9, 0xa9, 0x20, 0x16, 0x20, 0xdd, 0x64, 0x7f, 0x29, 0x99, 0xf3, 0x0a, 0x6c, + 0xc0, 0xdd, 0x9c, 0xc6, 0x61, 0xe7, 0xa7, 0xda, 0xdf, 0x05, 0x52, 0xe1, 0xa5, 0xad, 0xb9, 0x40, + 0x6a, 0xb0, 0x48, 0x31, 0xce, 0xa6, 0xb1, 0xf3, 0x91, 0xc8, 0x23, 0xfe, 0xcf, 0xa0, 0x09, 0xb5, + 0x67, 0xd3, 0xfe, 0x70, 0x32, 0x76, 0xaf, 0x98, 0xb9, 0x40, 0x00, 0xca, 0x58, 0x76, 0xb0, 0x81, + 0x69, 0xec, 0x6c, 0x03, 0xc4, 0xbf, 0x0e, 0x48, 0x15, 0x4a, 0x38, 0x32, 0x17, 0x48, 0x03, 0xaa, + 0xcf, 0x9d, 0x71, 0xf8, 0xdc, 0x71, 0x87, 0xa6, 0xb1, 0x73, 0x2f, 0x2e, 0x64, 0xd0, 0xe6, 0x95, + 0xef, 0x31, 0xe1, 0xed, 0xe9, 0x0d, 0x3a, 0x36, 0x76, 0xfe, 0x5d, 0x88, 0xbe, 0xa7, 0xa0, 0x1e, + 0x1d, 0x0b, 0x3f, 0xa2, 0xda, 0x32, 0x0d, 0xb2, 0xa4, 0x7e, 0xa6, 0x30, 0x0b, 0x84, 0x24, 0x3f, + 0x3b, 0x98, 0x45, 0x94, 0xe9, 0x0c, 0x6e, 0x96, 0x48, 0x7d, 0xf6, 0x49, 0xc1, 0x5c, 0x24, 0x77, + 0x32, 0x3e, 0x14, 0x98, 0x65, 0xb2, 0x0c, 0x75, 0xf9, 0x5f, 0x83, 0x4f, 0xaa, 0x90, 0x15, 0x68, + 0x4a, 0x81, 0xf4, 0x5f, 0x25, 0xab, 0xa9, 0x16, 0xde, 0xac, 0x11, 0x53, 0x6f, 0xc4, 0x4d, 0x40, + 0x89, 0x4a, 0x3b, 0x66, 0x1d, 0x7d, 0xa6, 0x2e, 0x66, 0xb3, 0x41, 0xd6, 0xb3, 0xba, 0x51, 0xb3, + 0x89, 0xe6, 0xa9, 0x0b, 0xd6, 0x5c, 0xc2, 0x25, 0x2a, 0x44, 0x62, 0x2e, 0xe3, 0x7a, 0x12, 0x87, + 0x6b, 0x9a, 0x64, 0x33, 0xaf, 0xf1, 0x33, 0x57, 0x76, 0x58, 0x54, 0xac, 0x88, 0x15, 0x71, 0x60, + 0xdc, 0xee, 0x33, 0x0f, 0x23, 0x61, 0x2e, 0xe0, 0x8a, 0x14, 0xb1, 0x8c, 0xac, 0x69, 0x28, 0xe6, + 0xaf, 0x79, 0xf0, 0xbb, 0x93, 0x7e, 0xdf, 0x2c, 0x28, 0xe2, 0x78, 0xbd, 0x66, 0xf1, 0x69, 0xe7, + 0xfb, 0xff, 0x6e, 0x19, 0xdf, 0xbd, 0xdb, 0x32, 0xbe, 0x7f, 0xb7, 0x65, 0xfc, 0xe7, 0xdd, 0xd6, + 0xc2, 0xdf, 0xff, 0xb7, 0x65, 0xfc, 0x5e, 0xfd, 0x3b, 0x76, 0xe9, 0x84, 0x81, 0x3b, 0xf5, 0x79, + 0x61, 0x1a, 0x0d, 0x3c, 0xb6, 0x3b, 0xba, 0x38, 0xdb, 0x1d, 0x9d, 0xee, 0xe2, 0x82, 0x4f, 0xcb, + 0xfc, 0x9f, 0xd8, 0xc3, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, 0xb0, 0xe6, 0x63, 0xfb, 0x67, 0x1b, + 0x00, 0x00, } func (m *LockOptions) Marshal() (dAtA []byte, err error) { @@ -4097,6 +4114,11 @@ func (m *GetBindResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.AllocatorVersion != 0 { + i = encodeVarintLock(dAtA, i, uint64(m.AllocatorVersion)) + i-- + dAtA[i] = 0x18 + } { size, err := m.LockTable.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -4196,6 +4218,11 @@ func (m *KeepLockTableBindResponse) MarshalToSizedBuffer(dAtA []byte) (int, erro i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.AllocatorVersion != 0 { + i = encodeVarintLock(dAtA, i, uint64(m.AllocatorVersion)) + i-- + dAtA[i] = 0x18 + } if m.Status != 0 { i = encodeVarintLock(dAtA, i, uint64(m.Status)) i-- @@ -5549,6 +5576,9 @@ func (m *GetBindResponse) ProtoSize() (n int) { _ = l l = m.LockTable.ProtoSize() n += 1 + l + sovLock(uint64(l)) + if m.AllocatorVersion != 0 { + n += 1 + sovLock(uint64(m.AllocatorVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5598,6 +5628,9 @@ func (m *KeepLockTableBindResponse) ProtoSize() (n int) { if m.Status != 0 { n += 1 + sovLock(uint64(m.Status)) } + if m.AllocatorVersion != 0 { + n += 1 + sovLock(uint64(m.AllocatorVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -9053,6 +9086,25 @@ func (m *GetBindResponse) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AllocatorVersion", wireType) + } + m.AllocatorVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLock + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AllocatorVersion |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipLock(dAtA[iNdEx:]) @@ -9311,6 +9363,25 @@ func (m *KeepLockTableBindResponse) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AllocatorVersion", wireType) + } + m.AllocatorVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLock + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AllocatorVersion |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipLock(dAtA[iNdEx:]) diff --git a/pkg/util/metric/v2/lockservice.go b/pkg/util/metric/v2/lockservice.go new file mode 100644 index 0000000000000..2c91488e5b708 --- /dev/null +++ b/pkg/util/metric/v2/lockservice.go @@ -0,0 +1,70 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v2 + +import "github.com/prometheus/client_golang/prometheus" + +var ( + lockServiceStaleBindPurgedCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "mo", + Subsystem: "lockservice", + Name: "stale_bind_purged_total", + Help: "Total number of stale lock table binds purged by lockservice allocator epoch fencing.", + }, []string{"source"}) + + lockServiceAllocatorEpochChangedCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "mo", + Subsystem: "lockservice", + Name: "allocator_epoch_changed_total", + Help: "Total number of allocator epoch changes observed by lockservice.", + }, []string{"source"}) + + lockServiceAllocatorEpochRegressionCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "mo", + Subsystem: "lockservice", + Name: "allocator_epoch_regression_total", + Help: "Total number of allocator epoch regressions observed by lockservice.", + }, []string{"source"}) + + LockServiceAllocatorEpochObservedGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "mo", + Subsystem: "lockservice", + Name: "allocator_epoch_observed", + Help: "Latest allocator epoch observed by lockservice.", + }) +) + +func GetLockServiceStaleBindPurgedCounter(source string) prometheus.Counter { + return lockServiceStaleBindPurgedCounter.WithLabelValues(source) +} + +func GetLockServiceAllocatorEpochChangedCounter(source string) prometheus.Counter { + return lockServiceAllocatorEpochChangedCounter.WithLabelValues(source) +} + +func GetLockServiceAllocatorEpochRegressionCounter(source string) prometheus.Counter { + return lockServiceAllocatorEpochRegressionCounter.WithLabelValues(source) +} + +func initLockServiceMetrics() { + registry.MustRegister(lockServiceStaleBindPurgedCounter) + registry.MustRegister(lockServiceAllocatorEpochChangedCounter) + registry.MustRegister(lockServiceAllocatorEpochRegressionCounter) + registry.MustRegister(LockServiceAllocatorEpochObservedGauge) +} diff --git a/pkg/util/metric/v2/metrics.go b/pkg/util/metric/v2/metrics.go index ee17323e36f31..0f4e6385e69b8 100644 --- a/pkg/util/metric/v2/metrics.go +++ b/pkg/util/metric/v2/metrics.go @@ -43,6 +43,7 @@ func init() { initFileServiceMetrics() initLogtailMetrics() initTxnMetrics() + initLockServiceMetrics() initTaskMetrics() initRPCMetrics() initMemMetrics() diff --git a/proto/lock.proto b/proto/lock.proto index 6282b9cc2470c..dd2a88ffd2ea9 100644 --- a/proto/lock.proto +++ b/proto/lock.proto @@ -252,6 +252,7 @@ message GetBindRequest { // GetBindResponse get bind request from allocator response. TN -> CN message GetBindResponse { LockTable LockTable = 2 [(gogoproto.nullable) = false]; + uint64 AllocatorVersion = 3; } // KeepLockTableBindRequest keep lock table bind request. CN -> TN @@ -266,6 +267,7 @@ message KeepLockTableBindRequest { message KeepLockTableBindResponse { bool OK = 1; Status Status = 2; + uint64 AllocatorVersion = 3; } // SetRestartServiceRequest set restart lock service request from allocator request. operator -> TN @@ -398,4 +400,4 @@ message ResumeInvalidCNRequest { message ResumeInvalidCNResponse { -} \ No newline at end of file +} From 3e51aa76baea27c9026bd71bca78da002be11ed4 Mon Sep 17 00:00:00 2001 From: iamlinjunhong <1030420200@qq.com> Date: Wed, 10 Jun 2026 10:19:45 +0800 Subject: [PATCH 2/6] test(lockservice): cover keepalive stale bind purge --- pkg/lockservice/service_test.go | 97 +++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/pkg/lockservice/service_test.go b/pkg/lockservice/service_test.go index ecea5843f529e..3d2f204ef5b58 100644 --- a/pkg/lockservice/service_test.go +++ b/pkg/lockservice/service_test.go @@ -3402,6 +3402,103 @@ func TestKeepaliveEpochPurgeKeepsGroupMovePop(t *testing.T) { ) } +func TestKeepaliveOKFalseWithNewAllocatorVersionPurgesStaleBinds(t *testing.T) { + runLockServiceTestsWithAdjustConfig( + t, + []string{"s1"}, + time.Second*10, + func(alloc *lockTableAllocator, s []*service) { + l1 := s[0] + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + alloc.mu.Lock() + oldVersion := alloc.version + alloc.mu.Unlock() + + l1.allocatorVersionMu.Lock() + l1.lastAllocatorVersion = oldVersion + l1.allocatorVersionMu.Unlock() + + staleLocalTable := uint64(21501) + staleRemoteTable := uint64(21502) + staleProxyTable := uint64(21503) + currentProxyTable := uint64(21504) + + staleLocalBind := pb.LockTable{ + Group: 0, + Table: staleLocalTable, + OriginTable: staleLocalTable, + ServiceID: l1.serviceID, + Version: oldVersion, + Valid: true, + } + l1.tableGroups.set(0, staleLocalTable, l1.createLockTableByBind(staleLocalBind)) + + staleRemoteBind := pb.LockTable{ + Group: 0, + Table: staleRemoteTable, + OriginTable: staleRemoteTable, + ServiceID: "remote-service", + Version: oldVersion, + Valid: true, + } + l1.tableGroups.set( + 0, + staleRemoteTable, + newRemoteLockTable( + l1.serviceID, + time.Second, + staleRemoteBind, + l1.remote.client, + l1.handleBindChanged, + l1.logger)) + + staleProxyBind := staleRemoteBind + staleProxyBind.Table = staleProxyTable + staleProxyBind.OriginTable = staleProxyTable + staleProxy := l1.createLockTableByBind(staleProxyBind) + _, ok := staleProxy.(*localLockTableProxy) + require.True(t, ok) + l1.tableGroups.set(0, staleProxyTable, staleProxy) + + alloc.mu.Lock() + alloc.mu.services = make(map[string]*serviceBinds) + alloc.mu.lockTables = make(map[uint32]map[uint64]pb.LockTable) + alloc.version++ + restartedVersion := alloc.version + alloc.mu.Unlock() + + currentProxyBind := staleRemoteBind + currentProxyBind.Table = currentProxyTable + currentProxyBind.OriginTable = currentProxyTable + currentProxyBind.Version = restartedVersion + currentProxy := l1.createLockTableByBind(currentProxyBind) + _, ok = currentProxy.(*localLockTableProxy) + require.True(t, ok) + l1.tableGroups.set(0, currentProxyTable, currentProxy) + + require.False(t, alloc.KeepLockTableBind(l1.serviceID)) + require.NotNil(t, l1.tableGroups.get(0, staleLocalTable)) + require.NotNil(t, l1.tableGroups.get(0, staleRemoteTable)) + require.NotNil(t, l1.tableGroups.get(0, staleProxyTable)) + require.NotNil(t, l1.tableGroups.get(0, currentProxyTable)) + + l1.remote.keeper.(*lockTableKeeper).doKeepLockTableBind(ctx) + + require.Nil(t, l1.tableGroups.get(0, staleLocalTable)) + require.Nil(t, l1.tableGroups.get(0, staleRemoteTable)) + require.Nil(t, l1.tableGroups.get(0, staleProxyTable)) + preserved := l1.tableGroups.get(0, currentProxyTable) + require.NotNil(t, preserved) + require.Equal(t, currentProxyBind, preserved.getBind()) + require.Equal(t, restartedVersion, l1.lastAllocatorVersion) + }, + func(cfg *Config) { + cfg.EnableRemoteLocalProxy = true + }) +} + func TestAllocatorObserverPurgesRemoteAndProxyLockTables(t *testing.T) { runLockServiceTestsWithAdjustConfig( t, From 79352ba9363ea5973bcb908e3aa454a39f4e7668 Mon Sep 17 00:00:00 2001 From: iamlinjunhong <1030420200@qq.com> Date: Wed, 10 Jun 2026 13:23:42 +0800 Subject: [PATCH 3/6] fix(lockservice): fence allocator replacements by instance id --- pkg/lockservice/lock_table_allocator.go | 9 + pkg/lockservice/lock_table_allocator_test.go | 8 +- pkg/lockservice/lock_table_keeper.go | 12 +- pkg/lockservice/lock_table_local.go | 4 +- pkg/lockservice/lock_table_proxy.go | 4 +- pkg/lockservice/lock_table_remote.go | 66 ++- pkg/lockservice/log.go | 44 +- pkg/lockservice/service.go | 220 +++++++-- pkg/lockservice/service_observability_test.go | 4 +- pkg/lockservice/service_remote.go | 103 ++++- pkg/lockservice/service_remote_test.go | 125 +++-- pkg/lockservice/service_test.go | 64 ++- pkg/lockservice/txn.go | 44 +- pkg/lockservice/types.go | 2 +- pkg/pb/lock/lock.go | 8 +- pkg/pb/lock/lock.pb.go | 431 ++++++++++++------ proto/lock.proto | 4 + 17 files changed, 885 insertions(+), 267 deletions(-) diff --git a/pkg/lockservice/lock_table_allocator.go b/pkg/lockservice/lock_table_allocator.go index 974b6bf022326..62c1a56602f1e 100644 --- a/pkg/lockservice/lock_table_allocator.go +++ b/pkg/lockservice/lock_table_allocator.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/google/uuid" "github.com/matrixorigin/matrixone/pkg/common/log" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/morpc" @@ -40,6 +41,7 @@ type lockTableAllocator struct { client Client inactiveService sync.Map // lock service id -> inactive time ctl sync.Map // lock service id -> *commitCtl + allocatorID string // version is the allocator process epoch. It is set once when the // allocator is constructed; production code must not mutate it at runtime. version uint64 @@ -85,6 +87,7 @@ func NewLockTableAllocator( stopper.WithLogger(logger.RawLogger().Named(tag))), keepBindTimeout: keepBindTimeout, client: rpcClient, + allocatorID: uuid.New().String(), version: uint64(time.Now().UnixNano()), } la.mu.lockTables = make(map[uint32]map[uint64]pb.LockTable) @@ -463,6 +466,9 @@ func (l *lockTableAllocator) tryRebindLocked( // If not a newer version of the same service, create new binding old.ServiceID = binds.serviceID + if old.AllocatorID == "" { + old.AllocatorID = l.allocatorID + } old.Version++ old.Valid = true l.getLockTablesLocked(group)[tableID] = old @@ -501,6 +507,7 @@ func (l *lockTableAllocator) createBindLocked( Valid: true, Sharding: sharding, Group: group, + AllocatorID: l.allocatorID, } l.getLockTablesLocked(group)[tableID] = b l.logger.Info("bind created", @@ -845,6 +852,7 @@ func (l *lockTableAllocator) handleGetBind( req *pb.Request, resp *pb.Response, cs morpc.ClientSession) { + resp.GetBind.AllocatorID = l.allocatorID resp.GetBind.AllocatorVersion = l.version if !l.canGetBind(req.GetBind.ServiceID) { writeResponse(l.logger, cancel, resp, moerr.NewNewTxnInCNRollingRestart(), cs) @@ -865,6 +873,7 @@ func (l *lockTableAllocator) handleKeepLockTableBind( req *pb.Request, resp *pb.Response, cs morpc.ClientSession) { + resp.KeepLockTableBind.AllocatorID = l.allocatorID resp.KeepLockTableBind.AllocatorVersion = l.version resp.KeepLockTableBind.OK = l.KeepLockTableBind(req.KeepLockTableBind.ServiceID) if !resp.KeepLockTableBind.OK { diff --git a/pkg/lockservice/lock_table_allocator_test.go b/pkg/lockservice/lock_table_allocator_test.go index 1d1068ed2ef7d..4ca20326b9f34 100644 --- a/pkg/lockservice/lock_table_allocator_test.go +++ b/pkg/lockservice/lock_table_allocator_test.go @@ -85,7 +85,7 @@ func TestGetWithNoBind(t *testing.T) { time.Hour, func(a *lockTableAllocator) { assert.Equal(t, - pb.LockTable{Valid: true, ServiceID: "s1", Table: 1, OriginTable: 1, Version: a.version}, + pb.LockTable{Valid: true, ServiceID: "s1", Table: 1, OriginTable: 1, Version: a.version, AllocatorID: a.allocatorID}, a.Get("s1", 0, 1, 0, pb.Sharding_None)) }) } @@ -98,7 +98,7 @@ func TestGetWithAlreadyBind(t *testing.T) { // register s1 first a.Get("s1", 0, 1, 0, pb.Sharding_None) assert.Equal(t, - pb.LockTable{Valid: true, ServiceID: "s1", Table: 1, OriginTable: 1, Version: a.version}, + pb.LockTable{Valid: true, ServiceID: "s1", Table: 1, OriginTable: 1, Version: a.version, AllocatorID: a.allocatorID}, a.Get("s2", 0, 1, 0, pb.Sharding_None)) }) } @@ -112,7 +112,7 @@ func TestGetWithBindInvalid(t *testing.T) { a.Get("s1", 0, 1, 0, pb.Sharding_None) a.disableTableBinds(a.getServiceBinds("s1")) assert.Equal(t, - pb.LockTable{Valid: true, ServiceID: "s2", Table: 1, OriginTable: 1, Version: a.version + 1}, + pb.LockTable{Valid: true, ServiceID: "s2", Table: 1, OriginTable: 1, Version: a.version + 1, AllocatorID: a.allocatorID}, a.Get("s2", 0, 1, 0, pb.Sharding_None)) }) } @@ -131,7 +131,7 @@ func TestGetWithBindAndServiceBothInvalid(t *testing.T) { a.getServiceBinds("s2").disable() assert.Equal(t, - pb.LockTable{Valid: false, ServiceID: "s1", Table: 1, OriginTable: 1, Version: a.version}, + pb.LockTable{Valid: false, ServiceID: "s1", Table: 1, OriginTable: 1, Version: a.version, AllocatorID: a.allocatorID}, a.Get("s2", 0, 1, 0, pb.Sharding_None)) }) } diff --git a/pkg/lockservice/lock_table_keeper.go b/pkg/lockservice/lock_table_keeper.go index 05da36d3b607a..d035bd60a20b7 100644 --- a/pkg/lockservice/lock_table_keeper.go +++ b/pkg/lockservice/lock_table_keeper.go @@ -196,9 +196,12 @@ func (k *lockTableKeeper) doKeepLockTableBind(ctx context.Context) { defer releaseResponse(resp) if resp.KeepLockTableBind.OK { - k.service.observeAllocatorVersionWithHolders( + k.service.observeAllocatorStateWithHolders( "keepalive", - resp.KeepLockTableBind.AllocatorVersion, + allocatorState{ + id: resp.KeepLockTableBind.AllocatorID, + version: resp.KeepLockTableBind.AllocatorVersion, + }, k.groupTables) switch resp.KeepLockTableBind.Status { case pb.Status_ServiceLockEnable: @@ -227,7 +230,10 @@ func (k *lockTableKeeper) doKeepLockTableBind(ctx context.Context) { k.serviceID, k.groupTables, oldVersion, - resp.KeepLockTableBind.AllocatorVersion) + allocatorState{ + id: resp.KeepLockTableBind.AllocatorID, + version: resp.KeepLockTableBind.AllocatorVersion, + }) if n > 0 { // Keep bind receiving an explicit failure means that all the binds of the local diff --git a/pkg/lockservice/lock_table_local.go b/pkg/lockservice/lock_table_local.go index 058489362405d..4b67d35efb5cf 100644 --- a/pkg/lockservice/lock_table_local.go +++ b/pkg/lockservice/lock_table_local.go @@ -390,7 +390,7 @@ func (l *localLockTable) getBind() pb.LockTable { return l.bind } -func (l *localLockTable) close() { +func (l *localLockTable) close(reasons ...closeReason) { l.mu.Lock() defer l.mu.Unlock() l.mu.closed = true @@ -405,7 +405,7 @@ func (l *localLockTable) close() { return true }) l.mu.store.Clear() - logLockTableClosed(l.logger, l.bind, false) + logLockTableClosed(l.logger, l.bind, false, closeReasonOrDefault(reasons)) } func (l *localLockTable) doAcquireLock(c *lockContext) error { diff --git a/pkg/lockservice/lock_table_proxy.go b/pkg/lockservice/lock_table_proxy.go index aac797d63b501..46c4d837a74d7 100644 --- a/pkg/lockservice/lock_table_proxy.go +++ b/pkg/lockservice/lock_table_proxy.go @@ -206,8 +206,8 @@ func (lp *localLockTableProxy) getBind() pb.LockTable { return lp.remote.getBind() } -func (lp *localLockTableProxy) close() { - lp.remote.close() +func (lp *localLockTableProxy) close(reasons ...closeReason) { + lp.remote.close(reasons...) } type sharedOps struct { diff --git a/pkg/lockservice/lock_table_remote.go b/pkg/lockservice/lock_table_remote.go index b7fd085037b0e..e0d04ef892937 100644 --- a/pkg/lockservice/lock_table_remote.go +++ b/pkg/lockservice/lock_table_remote.go @@ -135,10 +135,25 @@ func (l *remoteLockTable) lock( cb(pb.Result{}, ErrTxnNotFound) return } + if txn.bindChanged { + cb(pb.Result{}, ErrLockTableBindChanged) + return + } if err == nil { defer releaseResponse(resp) - if err := l.maybeHandleBindChanged(resp); err != nil { + if resp.NewBind != nil { + txn.Unlock() + err = l.maybeHandleBindChanged(resp) + txn.Lock() + if !bytes.Equal(req.Lock.TxnID, txn.txnID) { + cb(pb.Result{}, ErrTxnNotFound) + return + } + if txn.bindChanged { + cb(pb.Result{}, ErrLockTableBindChanged) + return + } logRemoteLockFailed(l.logger, txn, rows, opts, l.bind, err) cb(pb.Result{}, err) return @@ -150,14 +165,29 @@ func (l *remoteLockTable) lock( return } - // encounter any error, we also added lock to txn, because we need unlock on remote - _ = txn.lockAdded(l.bind.Group, l.bind, rows, l.logger) + // Only record the lock on errors when the request could have been observed + // by the remote side. For local context timeouts / cancellations, the + // request may never be sent, and recording a phantom lock will leak waiters. + if !skipTrackLockOnError(ctx, err) { + _ = txn.lockAdded(l.bind.Group, l.bind, rows, l.logger) + } logRemoteLockFailed(l.logger, txn, rows, opts, l.bind, err) // encounter any error, we need try to check bind is valid. // And use origin error to return, because once handlerError // swallows the error, the transaction will not be abort. originalErr := err - if e := l.handleError(err, true); e != nil { + txn.Unlock() + e := l.handleError(err, true) + txn.Lock() + if !bytes.Equal(req.Lock.TxnID, txn.txnID) { + cb(pb.Result{}, ErrTxnNotFound) + return + } + if txn.bindChanged { + cb(pb.Result{}, ErrLockTableBindChanged) + return + } + if e != nil { err = e } else { // handleError returned nil, meaning bind changed and error was swallowed @@ -318,8 +348,8 @@ func (l *remoteLockTable) getBind() pb.LockTable { return l.bind } -func (l *remoteLockTable) close() { - logLockTableClosed(l.logger, l.bind, true) +func (l *remoteLockTable) close(reasons ...closeReason) { + logLockTableClosed(l.logger, l.bind, true, closeReasonOrDefault(reasons)) } func (l *remoteLockTable) handleError( @@ -358,6 +388,30 @@ func (l *remoteLockTable) handleError( return oldError } +// skipTrackLockOnError returns true when the error indicates the request +// definitely did not reach the remote lock service (e.g. local context +// cancellation/timeout). In such cases, recording the lock would create a +// phantom holder that can never be cleaned up. +// +// If the request actually reached the remote side before timeout, the remote +// lock will be cleaned up by the keepRemoteLock timeout mechanism (bounded by +// RemoteLockTimeout, default 10m), so skipping the local bookkeeping is safe +// though it may introduce a bounded extra wait. +func skipTrackLockOnError(ctx context.Context, err error) bool { + if ctxErr := ctx.Err(); ctxErr != nil { + if errors.Is(ctxErr, context.DeadlineExceeded) || + errors.Is(ctxErr, context.Canceled) { + return true + } + } + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) || + moerr.IsMoErrCode(err, moerr.ErrRPCTimeout) { + return true + } + return false +} + func retryRemoteLockError(err error) bool { var netErr net.Error if errors.As(err, &netErr) && netErr.Timeout() { diff --git a/pkg/lockservice/log.go b/pkg/lockservice/log.go index a5bd62ec73dae..674d745dbdc4d 100644 --- a/pkg/lockservice/log.go +++ b/pkg/lockservice/log.go @@ -392,19 +392,61 @@ func logLockTableClosed( logger *log.MOLogger, bind pb.LockTable, remote bool, + reason closeReason, ) { if logger == nil { return } + logLevel := zap.InfoLevel + if reason == closeReasonServiceClose { + logLevel = zap.DebugLevel + if !logger.Enabled(logLevel) { + return + } + } + logger.Log( "bind closed", - getLogOptions(zap.InfoLevel), + getLogOptions(logLevel), zap.Bool("remote", remote), zap.String("bind", bind.DebugString()), + zap.String("reason", reason.String()), ) } +// closeReason represents the reason why a lock table is closed. +type closeReason int + +const ( + closeReasonBindChanged closeReason = iota + closeReasonServiceClose + closeReasonKeeperFailed + closeReasonKeepBindFailed +) + +func (r closeReason) String() string { + switch r { + case closeReasonBindChanged: + return "bind-changed" + case closeReasonServiceClose: + return "service-close" + case closeReasonKeeperFailed: + return "keeper-failed" + case closeReasonKeepBindFailed: + return "keep-bind-failed" + default: + return "unknown" + } +} + +func closeReasonOrDefault(reasons []closeReason) closeReason { + if len(reasons) == 0 { + return closeReasonBindChanged + } + return reasons[0] +} + func logDeadLockFound( logger *log.MOLogger, txn pb.WaitTxn, diff --git a/pkg/lockservice/service.go b/pkg/lockservice/service.go index 07f2c0ea69b91..f34c0e62a4fae 100644 --- a/pkg/lockservice/service.go +++ b/pkg/lockservice/service.go @@ -57,11 +57,13 @@ type service struct { clock clock.Clock stopper *stopper.Stopper stopOnce sync.Once + bindChangeMu sync.RWMutex fetchWhoWaitingListC chan who logger *log.MOLogger allocatorVersionMu sync.Mutex lastAllocatorVersion uint64 + lastAllocatorID string remote struct { client Client @@ -163,28 +165,43 @@ func (s *service) Lock( return pb.Result{}, err } + s.bindChangeMu.RLock() // All txn lock op must be serial. And avoid dead lock between doAcquireLock // and getLock. The doAcquireLock and getLock operations of the same transaction // will be concurrent (deadlock detection), which may lead to a deadlock in mutex. txn.Lock() defer txn.Unlock() if !bytes.Equal(txn.txnID, txnID) { + s.bindChangeMu.RUnlock() return pb.Result{}, ErrTxnNotFound } if txn.deadlockFound { + s.bindChangeMu.RUnlock() return pb.Result{}, ErrDeadLockDetected } + if txn.bindChanged { + s.bindChangeMu.RUnlock() + return pb.Result{}, ErrLockTableBindChanged + } // it needs to inc table bind ref when set restart cn - h := txn.getHoldLocksLocked(l.getBind().Group) - _, hasBind := h.tableBinds[l.getBind().Table] + bind := l.getBind() + current := s.tableGroups.get(bind.Group, bind.Table) + if current == nil || current.getBind().Changed(bind) { + s.bindChangeMu.RUnlock() + return pb.Result{}, ErrLockTableBindChanged + } + h := txn.getHoldLocksLocked(bind.Group) + _, hasBind := h.tableBinds[bind.Table] + txn.lockTableBindTouched(bind) + s.bindChangeMu.RUnlock() defer func() { if s.isStatus(pb.Status_ServiceLockEnable) || err != nil || hasBind { return } - s.incRef(l.getBind().Group, l.getBind().Table) + s.incRef(bind.Group, bind.Table) }() var result pb.Result @@ -197,9 +214,40 @@ func (s *service) Lock( result = r err = e }) + if err == nil { + if e := s.checkBindChangedBeforeLockSuccess(txn, txnID, bind); e != nil { + result = pb.Result{} + err = e + } + } return result, err } +func (s *service) checkBindChangedBeforeLockSuccess( + txn *activeTxn, + txnID []byte, + bind pb.LockTable, +) error { + // Let any pending bind-change fence complete before reporting lock success. + // Keep the lock order consistent with Lock: bindChangeMu before txn.Lock. + txn.Unlock() + s.bindChangeMu.RLock() + txn.Lock() + defer s.bindChangeMu.RUnlock() + + if !bytes.Equal(txn.txnID, txnID) { + return ErrTxnNotFound + } + if txn.bindChanged { + return ErrLockTableBindChanged + } + l := s.tableGroups.get(bind.Group, bind.Table) + if l == nil || l.getBind().Changed(bind) { + return ErrLockTableBindChanged + } + return nil +} + func (s *service) Unlock( ctx context.Context, txnID []byte, @@ -394,7 +442,7 @@ func (s *service) Close() error { var err error s.stopOnce.Do(func() { s.stopper.Stop() - s.tableGroups.removeWithFilter(func(_ uint64, _ lockTable) bool { return true }) + s.tableGroups.removeWithFilter(func(_ uint64, _ lockTable) bool { return true }, closeReasonServiceClose) if err = s.remote.client.Close(); err != nil { return } @@ -588,7 +636,7 @@ func (s *service) getLockTableWithCreate( close(c) }() - bind, allocatorVersion, err := getLockTableBind( + bind, allocator, err := getLockTableBind( s.remote.client, group, tableID, @@ -599,18 +647,33 @@ func (s *service) getLockTableWithCreate( return nil, err } - s.observeAllocatorVersion("get-bind", allocatorVersion) + s.observeAllocatorState("get-bind", allocator) v := s.tableGroups.set(group, tableID, s.createLockTableByBind(bind)) return v, nil } func (s *service) handleBindChanged(newBind pb.LockTable) { + s.bindChangeMu.Lock() + defer s.bindChangeMu.Unlock() + new := s.createLockTableByBind(newBind) s.tableGroups.set(newBind.Group, newBind.Table, new) + s.fenceByBindChanged(newBind) +} + +func (s *service) fenceByBindChanged(bind pb.LockTable) { + if s.activeTxnHolder == nil { + return + } + s.activeTxnHolder.fenceByBindChanged(bind) } func (s *service) observeAllocatorVersion(source string, observedVersion uint64) int { - return s.observeAllocatorVersionWithHolders(source, observedVersion, s.tableGroups) + return s.observeAllocatorStateWithHolders(source, allocatorState{version: observedVersion}, s.tableGroups) +} + +func (s *service) observeAllocatorState(source string, observed allocatorState) int { + return s.observeAllocatorStateWithHolders(source, observed, s.tableGroups) } func (s *service) observeAllocatorVersionWithHolders( @@ -618,83 +681,144 @@ func (s *service) observeAllocatorVersionWithHolders( observedVersion uint64, holders *lockTableHolders, ) int { - if observedVersion == 0 { - return 0 - } + return s.observeAllocatorStateWithHolders(source, allocatorState{version: observedVersion}, holders) +} +func (s *service) observeAllocatorStateWithHolders( + source string, + observed allocatorState, + holders *lockTableHolders, +) int { s.allocatorVersionMu.Lock() defer s.allocatorVersionMu.Unlock() - return s.observeAllocatorVersionLocked(source, observedVersion, holders) + return s.observeAllocatorStateLocked(source, observed, holders) } -func (s *service) observeAllocatorVersionLocked( +func (s *service) observeAllocatorStateLocked( source string, - observedVersion uint64, + observed allocatorState, holders *lockTableHolders, ) int { - if observedVersion == 0 { + if observed.version == 0 && observed.id == "" { return 0 } oldVersion := s.lastAllocatorVersion - if oldVersion != 0 && observedVersion < oldVersion { + oldID := s.lastAllocatorID + allocatorChanged := observed.id != "" && + observed.id != oldID && + (oldID != "" || oldVersion != 0) + if !allocatorChanged && oldVersion != 0 && observed.version < oldVersion { v2.GetLockServiceAllocatorEpochRegressionCounter(source).Inc() - logAllocatorEpochRegression(s.logger, source, oldVersion, observedVersion) + logAllocatorEpochRegression(s.logger, source, oldVersion, observed.version) return 0 } - if observedVersion == oldVersion { + if !allocatorChanged && observed.version == oldVersion { return 0 } // Defensive path for minimal service instances used by keeper tests. // Normal lock services always pass a holder and can purge stale binds. if holders == nil { - s.lastAllocatorVersion = observedVersion - v2.LockServiceAllocatorEpochObservedGauge.Set(float64(observedVersion)) + s.updateAllocatorStateLocked(observed) + v2.LockServiceAllocatorEpochObservedGauge.Set(float64(observed.version)) v2.GetLockServiceAllocatorEpochChangedCounter(source).Inc() - logAllocatorEpochChanged(s.logger, source, oldVersion, observedVersion, 0) + logAllocatorEpochChanged(s.logger, source, oldVersion, observed.version, 0) return 0 } - removed := holders.removeWithFilter(func(_ uint64, lt lockTable) bool { - return lt.getBind().Version < observedVersion - }) - s.lastAllocatorVersion = observedVersion - v2.LockServiceAllocatorEpochObservedGauge.Set(float64(observedVersion)) + s.bindChangeMu.Lock() + removed := s.removeLockTablesWithFence(holders, func(bind pb.LockTable) bool { + if allocatorChanged { + if observed.id != "" && bind.AllocatorID != "" { + return bind.AllocatorID != observed.id + } + return true + } + return bind.Version < observed.version + }, observed) + s.bindChangeMu.Unlock() + + s.updateAllocatorStateLocked(observed) + v2.LockServiceAllocatorEpochObservedGauge.Set(float64(observed.version)) v2.GetLockServiceAllocatorEpochChangedCounter(source).Inc() if removed > 0 { v2.GetLockServiceStaleBindPurgedCounter(source).Add(float64(removed)) } - logAllocatorEpochChanged(s.logger, source, oldVersion, observedVersion, removed) + logAllocatorEpochChanged(s.logger, source, oldVersion, observed.version, removed) + return removed +} + +func (s *service) updateAllocatorStateLocked(observed allocatorState) { + if observed.id != "" { + s.lastAllocatorID = observed.id + } + if observed.version != 0 { + s.lastAllocatorVersion = observed.version + } +} + +func (s *service) removeLockTablesWithFence( + holders *lockTableHolders, + filter func(pb.LockTable) bool, + observed allocatorState, +) int { + var removedBinds []pb.LockTable + removed := holders.removeWithFilter(func(_ uint64, lt lockTable) bool { + bind := lt.getBind() + if !filter(bind) { + return false + } + removedBinds = append(removedBinds, fenceBindForAllocatorState(bind, observed)) + return true + }, closeReasonBindChanged) + for _, bind := range removedBinds { + s.fenceByBindChanged(bind) + } return removed } +func fenceBindForAllocatorState( + bind pb.LockTable, + observed allocatorState, +) pb.LockTable { + bind.AllocatorID = observed.id + if observed.version > bind.Version { + bind.Version = observed.version + } else { + bind.Version++ + } + return bind +} + func (s *service) handleKeepBindFailed( serviceID string, holders *lockTableHolders, - oldTableVersion, - allocatorVersion uint64, + oldTableVersion uint64, + allocator allocatorState, ) int { s.allocatorVersionMu.Lock() defer s.allocatorVersionMu.Unlock() if holders == nil { - s.observeAllocatorVersionLocked("keepalive-ok-false-epoch", allocatorVersion, nil) + s.observeAllocatorStateLocked("keepalive-ok-false-epoch", allocator, nil) return 0 } // Keep the original OK=false snapshot guard: if another local purge already // changed holders.version, this service-level purge should skip this round. + s.bindChangeMu.Lock() removed := holders.removeWithFilter(func(_ uint64, lt lockTable) bool { if oldTableVersion != holders.getVersion() { return false } return lt.getBind().ServiceID == serviceID - }) + }, closeReasonKeepBindFailed) + s.bindChangeMu.Unlock() if removed > 0 { v2.GetLockServiceStaleBindPurgedCounter("keepalive-ok-false-service").Add(float64(removed)) } - s.observeAllocatorVersionLocked("keepalive-ok-false-epoch", allocatorVersion, holders) + s.observeAllocatorStateLocked("keepalive-ok-false-epoch", allocator, holders) return removed } @@ -746,6 +870,7 @@ type activeTxnHolder interface { getActiveTxn(txnID []byte, create bool, remoteService string) *activeTxn hasActiveTxn(txnID []byte) bool deleteActiveTxn(txnID []byte) *activeTxn + fenceByBindChanged(bind pb.LockTable) int keepRemoteActiveTxn(remoteService string) keepRemoteLockBindActive(remoteService string, bind pb.LockTable) hasRemoteLockBind(remoteService string, bind pb.LockTable, maxKeepInterval time.Duration) bool @@ -887,6 +1012,19 @@ func (h *mapBasedTxnHolder) deleteActiveTxn(txnID []byte) *activeTxn { return v } +func (h *mapBasedTxnHolder) fenceByBindChanged(bind pb.LockTable) int { + h.mu.RLock() + defer h.mu.RUnlock() + + n := 0 + for _, txn := range h.mu.activeTxns { + if txn.fenceByBindChanged(bind, h.logger) { + n++ + } + } + return n +} + func (h *mapBasedTxnHolder) keepRemoteActiveTxn(remoteService string) { h.mu.Lock() defer h.mu.Unlock() @@ -1158,13 +1296,17 @@ func (m *lockTableHolders) iter(fn func(uint64, lockTable) bool) { } } -func (m *lockTableHolders) removeWithFilter(filter func(uint64, lockTable) bool) int { +func (m *lockTableHolders) removeWithFilter( + filter func(uint64, lockTable) bool, + reasons ...closeReason, +) int { m.RLock() defer m.RUnlock() + reason := closeReasonOrDefault(reasons) removed := 0 for _, h := range m.holders { - removed += h.removeWithFilter(filter) + removed += h.removeWithFilter(filter, reason) } if removed > 0 { m.version.Add(1) @@ -1207,12 +1349,12 @@ func (m *lockTableHolder) set( oldBind := old.getBind() newBind := new.getBind() if oldBind.Changed(newBind) { - old.close() + old.close(closeReasonBindChanged) m.tables[id] = new logRemoteBindChanged(logger, m.service, oldBind, newBind) return new } - new.close() + new.close(closeReasonBindChanged) return old } @@ -1227,13 +1369,17 @@ func (m *lockTableHolder) iter(fn func(uint64, lockTable) bool) bool { return true } -func (m *lockTableHolder) removeWithFilter(filter func(uint64, lockTable) bool) int { +func (m *lockTableHolder) removeWithFilter( + filter func(uint64, lockTable) bool, + reasons ...closeReason, +) int { m.Lock() defer m.Unlock() + reason := closeReasonOrDefault(reasons) removed := 0 for id, v := range m.tables { if filter(id, v) { - v.close() + v.close(reason) delete(m.tables, id) removed++ } diff --git a/pkg/lockservice/service_observability_test.go b/pkg/lockservice/service_observability_test.go index a3c4f412a779a..c9437ae60dc0c 100644 --- a/pkg/lockservice/service_observability_test.go +++ b/pkg/lockservice/service_observability_test.go @@ -97,7 +97,7 @@ func TestForceRefreshLockTableBinds(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, _ *service, table uint64) { l1.ForceRefreshLockTableBinds(nil, nil) l2.ForceRefreshLockTableBinds(nil, nil) @@ -132,7 +132,7 @@ func TestGetLockTableBind(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, _ *service, table uint64) { bind1, err := l1.GetLockTableBind(0, table) require.NoError(t, err) diff --git a/pkg/lockservice/service_remote.go b/pkg/lockservice/service_remote.go index 103c57d8a200b..014128482180a 100644 --- a/pkg/lockservice/service_remote.go +++ b/pkg/lockservice/service_remote.go @@ -184,8 +184,9 @@ func (s *service) handleRemoteLock( req *pb.Request, resp *pb.Response, cs morpc.ClientSession) { + logFields := remoteLockResponseLogFields(req) if !s.canLockOnServiceStatus(req.Lock.TxnID, req.Lock.Options, req.LockTable.Table, req.Lock.Rows) { - _ = writeResponseWithDeadline(s.logger, cancel, resp, moerr.NewRetryForCNRollingRestart(), cs, defaultRPCWriteTimeout, remoteLockResponseLogFields(req)) + _ = writeResponseWithDeadline(s.logger, cancel, resp, moerr.NewRetryForCNRollingRestart(), cs, defaultRPCWriteTimeout, logFields) return } @@ -194,7 +195,23 @@ func (s *service) handleRemoteLock( l == nil { // means that the lockservice sending the lock request holds a stale // lock table binding. - _ = writeResponseWithDeadline(s.logger, cancel, resp, err, cs, defaultRPCWriteTimeout, remoteLockResponseLogFields(req)) + _ = writeResponseWithDeadline(s.logger, cancel, resp, err, cs, defaultRPCWriteTimeout, logFields) + return + } + + s.bindChangeMu.RLock() + bind := l.getBind() + current := s.tableGroups.get(bind.Group, bind.Table) + if current == nil { + s.bindChangeMu.RUnlock() + _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrLockTableNotFound, cs, defaultRPCWriteTimeout, logFields) + return + } + if current.getBind().Changed(bind) { + newBind := current.getBind() + resp.NewBind = &newBind + s.bindChangeMu.RUnlock() + _ = writeResponseWithDeadline(s.logger, cancel, resp, nil, cs, defaultRPCWriteTimeout, logFields) return } s.activeTxnHolder.keepRemoteActiveTxn(req.Lock.ServiceID) @@ -202,29 +219,41 @@ func (s *service) handleRemoteLock( txn := s.activeTxnHolder.getActiveTxn(req.Lock.TxnID, true, req.Lock.ServiceID) txn.Lock() - defer txn.Unlock() if !bytes.Equal(txn.txnID, req.Lock.TxnID) { - _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrTxnNotFound, cs, defaultRPCWriteTimeout, remoteLockResponseLogFields(req)) + txn.Unlock() + s.bindChangeMu.RUnlock() + _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrTxnNotFound, cs, defaultRPCWriteTimeout, logFields) return } if txn.deadlockFound { - _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrDeadLockDetected, cs, defaultRPCWriteTimeout, remoteLockResponseLogFields(req)) + txn.Unlock() + s.bindChangeMu.RUnlock() + _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrDeadLockDetected, cs, defaultRPCWriteTimeout, logFields) + return + } + if txn.bindChanged { + txn.Unlock() + s.bindChangeMu.RUnlock() + _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrLockTableBindChanged, cs, defaultRPCWriteTimeout, logFields) return } var lockErr error // it needs to inc table bind ref when set restart cn - h := txn.getHoldLocksLocked(l.getBind().Group) - _, hasBind := h.tableBinds[l.getBind().Table] + h := txn.getHoldLocksLocked(bind.Group) + _, hasBind := h.tableBinds[bind.Table] + txn.lockTableBindTouched(bind) + txnID := append([]byte(nil), req.Lock.TxnID...) + s.bindChangeMu.RUnlock() + defer txn.Unlock() defer func() { if s.isStatus(pb.Status_ServiceLockEnable) || lockErr != nil || hasBind { return } - s.incRef(l.getBind().Group, l.getBind().Table) + s.incRef(bind.Group, bind.Table) }() - logFields := remoteLockResponseLogFieldsFromValues(txn.txnID, l.getBind(), txn.remoteService, len(req.Lock.Rows)) l.lock( ctx, @@ -232,6 +261,12 @@ func (s *service) handleRemoteLock( req.Lock.Rows, LockOptions{LockOptions: req.Lock.Options, async: true}, func(result pb.Result, err error) { + if err == nil { + if e := s.checkBindChangedBeforeLockSuccess(txn, txnID, bind); e != nil { + result = pb.Result{} + err = e + } + } lockErr = err resp.Lock.Result = result _ = writeResponseWithDeadline(s.logger, cancel, resp, err, cs, defaultRPCWriteTimeout, logFields) @@ -244,11 +279,13 @@ func (s *service) handleForwardLock( req *pb.Request, resp *pb.Response, cs morpc.ClientSession) { + logFields := remoteLockResponseLogFields(req) if !s.canLockOnServiceStatus(req.Lock.TxnID, req.Lock.Options, req.LockTable.Table, req.Lock.Rows) { - _ = writeResponseWithDeadline(s.logger, cancel, resp, moerr.NewRetryForCNRollingRestart(), cs, defaultRPCWriteTimeout, remoteLockResponseLogFields(req)) + _ = writeResponseWithDeadline(s.logger, cancel, resp, moerr.NewRetryForCNRollingRestart(), cs, defaultRPCWriteTimeout, logFields) return } + s.bindChangeMu.RLock() l, err := s.getLockTable( req.LockTable.Group, req.LockTable.Table) @@ -256,7 +293,8 @@ func (s *service) handleForwardLock( l == nil { // means that the lockservice sending the lock request holds a stale // lock table binding. - _ = writeResponseWithDeadline(s.logger, cancel, resp, err, cs, defaultRPCWriteTimeout, remoteLockResponseLogFields(req)) + s.bindChangeMu.RUnlock() + _ = writeResponseWithDeadline(s.logger, cancel, resp, err, cs, defaultRPCWriteTimeout, logFields) return } @@ -266,28 +304,39 @@ func (s *service) handleForwardLock( txn.Lock() if !bytes.Equal(txn.txnID, req.Lock.TxnID) { txn.Unlock() - _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrTxnNotFound, cs, defaultRPCWriteTimeout, remoteLockResponseLogFields(req)) + s.bindChangeMu.RUnlock() + _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrTxnNotFound, cs, defaultRPCWriteTimeout, logFields) return } if txn.deadlockFound { txn.Unlock() - _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrDeadLockDetected, cs, defaultRPCWriteTimeout, remoteLockResponseLogFields(req)) + s.bindChangeMu.RUnlock() + _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrDeadLockDetected, cs, defaultRPCWriteTimeout, logFields) + return + } + if txn.bindChanged { + txn.Unlock() + s.bindChangeMu.RUnlock() + _ = writeResponseWithDeadline(s.logger, cancel, resp, ErrLockTableBindChanged, cs, defaultRPCWriteTimeout, logFields) return } var lockErr error // it needs to inc table bind ref when set restart cn - h := txn.getHoldLocksLocked(l.getBind().Group) - _, hasBind := h.tableBinds[l.getBind().Table] + bind := l.getBind() + h := txn.getHoldLocksLocked(bind.Group) + _, hasBind := h.tableBinds[bind.Table] + txn.lockTableBindTouched(bind) + txnID := append([]byte(nil), req.Lock.TxnID...) + s.bindChangeMu.RUnlock() defer func() { if s.isStatus(pb.Status_ServiceLockEnable) || lockErr != nil || hasBind { return } - s.incRef(l.getBind().Group, l.getBind().Table) + s.incRef(bind.Group, bind.Table) }() - logFields := remoteLockResponseLogFieldsFromValues(txn.txnID, l.getBind(), txn.remoteService, len(req.Lock.Rows)) l.lock( ctx, @@ -295,6 +344,12 @@ func (s *service) handleForwardLock( req.Lock.Rows, LockOptions{LockOptions: req.Lock.Options, async: true}, func(result pb.Result, err error) { + if err == nil { + if e := s.checkBindChangedBeforeLockSuccess(txn, txnID, bind); e != nil { + result = pb.Result{} + err = e + } + } txn.Unlock() lockErr = err resp.Lock.Result = result @@ -633,13 +688,18 @@ func (s *service) isValidLocalTxn(t []byte) bool { return len(committing) != 0 } +type allocatorState struct { + id string + version uint64 +} + func getLockTableBind( c Client, group uint32, tableID uint64, originTableID uint64, serviceID string, - sharding pb.Sharding) (pb.LockTable, uint64, error) { + sharding pb.Sharding) (pb.LockTable, allocatorState, error) { ctx, cancel := context.WithTimeoutCause(context.Background(), defaultRPCTimeout, moerr.CauseGetLockTableBind) defer cancel() @@ -655,11 +715,14 @@ func getLockTableBind( resp, err := c.Send(ctx, req) if err != nil { - return pb.LockTable{}, 0, moerr.AttachCause(ctx, err) + return pb.LockTable{}, allocatorState{}, moerr.AttachCause(ctx, err) } defer releaseResponse(resp) v := resp.GetBind.LockTable - return v, resp.GetBind.AllocatorVersion, nil + return v, allocatorState{ + id: resp.GetBind.AllocatorID, + version: resp.GetBind.AllocatorVersion, + }, nil } type who struct { diff --git a/pkg/lockservice/service_remote_test.go b/pkg/lockservice/service_remote_test.go index e45196e2c948e..44d217f6dcfd4 100644 --- a/pkg/lockservice/service_remote_test.go +++ b/pkg/lockservice/service_remote_test.go @@ -313,7 +313,7 @@ func TestLockWithBindIsStale(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, _ *service, table uint64) { txnID2 := []byte("txn2") _, err := l2.Lock(ctx, table, [][]byte{{3}}, txnID2, pb.LockOptions{ @@ -327,7 +327,7 @@ func TestLockWithBindIsStale(t *testing.T) { checkBind( t, - pb.LockTable{ServiceID: l1.serviceID, Version: alloc.version + 1, Table: table, OriginTable: table, Valid: true}, + pb.LockTable{ServiceID: l1.serviceID, Version: alloc.version + 1, Table: table, OriginTable: table, Valid: true, AllocatorID: alloc.allocatorID}, l2) }, ) @@ -340,16 +340,21 @@ func TestUnlockWithBindIsStable(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, _ *service, table uint64) { txnID2 := []byte("txn2") l2.Unlock(ctx, txnID2, timestamp.Timestamp{}) - checkBind( - t, - pb.LockTable{ServiceID: l1.serviceID, Version: alloc.version, Table: table, OriginTable: table, Valid: true}, - l2) + bind := l2.tableGroups.get(0, table).getBind() + require.Equal(t, l1.serviceID, bind.ServiceID) + require.Equal(t, table, bind.Table) + require.Equal(t, table, bind.OriginTable) + require.True(t, bind.Valid) + require.True(t, + bind.Version == alloc.version || bind.Version == alloc.version+1, + "bind can stay unchanged or be refreshed asynchronously by keepRemoteLock, got %s", + bind.DebugString()) }, ) } @@ -361,7 +366,7 @@ func TestGetLockWithBindIsStable(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, _ *service, table uint64) { txnID2 := []byte("txn2") @@ -371,7 +376,7 @@ func TestGetLockWithBindIsStable(t *testing.T) { checkBind( t, - pb.LockTable{ServiceID: l1.serviceID, Version: alloc.version + 1, Table: table, OriginTable: table, Valid: true}, + pb.LockTable{ServiceID: l1.serviceID, Version: alloc.version + 1, Table: table, OriginTable: table, Valid: true, AllocatorID: alloc.allocatorID}, l2) }, ) @@ -384,7 +389,7 @@ func TestLockWithBindTimeout(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, _ *service, table uint64) { // stop l1 let old bind invalid require.NoError(t, l1.Close()) @@ -392,21 +397,34 @@ func TestLockWithBindTimeout(t *testing.T) { waitBindDisabled(t, alloc, l1.serviceID) txnID2 := []byte("txn2") - // l2 hold the old bind, and can not connect to s1, and wait bind changed - for { - _, err := l2.Lock(ctx, table, [][]byte{{3}}, txnID2, pb.LockOptions{ - Granularity: pb.Granularity_Row, - Mode: pb.LockMode_Exclusive, - Policy: pb.WaitPolicy_Wait, - }) - if err == nil { - // l2 get the bind - l := l2.tableGroups.get(0, table) - assert.Equal(t, l2.serviceID, l.getBind().ServiceID) - return - } - time.Sleep(time.Millisecond * 100) - } + // l2 holds the old bind. Once bind change is detected, the old txn is fenced + // and the caller must retry with a new txn. + _, err := l2.Lock(ctx, table, [][]byte{{3}}, txnID2, pb.LockOptions{ + Granularity: pb.Granularity_Row, + Mode: pb.LockMode_Exclusive, + Policy: pb.WaitPolicy_Wait, + }) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrLockTableBindChanged)) + + _, err = l2.Lock(ctx, table, [][]byte{{3}}, txnID2, pb.LockOptions{ + Granularity: pb.Granularity_Row, + Mode: pb.LockMode_Exclusive, + Policy: pb.WaitPolicy_Wait, + }) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrLockTableBindChanged)) + + txnID3 := []byte("txn3") + _, err = l2.Lock(ctx, table, [][]byte{{3}}, txnID3, pb.LockOptions{ + Granularity: pb.Granularity_Row, + Mode: pb.LockMode_Exclusive, + Policy: pb.WaitPolicy_Wait, + }) + require.NoError(t, err) + require.NoError(t, l2.Unlock(ctx, txnID2, timestamp.Timestamp{})) + require.NoError(t, l2.Unlock(ctx, txnID3, timestamp.Timestamp{})) + + l := l2.tableGroups.get(0, table) + assert.Equal(t, l2.serviceID, l.getBind().ServiceID) }, ) } @@ -418,7 +436,7 @@ func TestUnlockWithBindTimeout(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, _ *service, table uint64) { // stop l1 let old bind invalid require.NoError(t, l1.Close()) @@ -441,7 +459,7 @@ func TestGetLockWithBindTimeout(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, _ *service, table uint64) { // stop l1 let old bind invalid require.NoError(t, l1.Close()) @@ -466,11 +484,17 @@ func TestLockWithBindNotFound(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, l3 *service, table uint64) { // change l2's bind to s3, no bind in s3 - l2.handleBindChanged(pb.LockTable{Table: table, ServiceID: "s3", Valid: true, Version: alloc.version + 1}) + l2.handleBindChanged(pb.LockTable{ + Table: table, + ServiceID: l3.serviceID, + Valid: true, + Version: alloc.version + 1, + OriginTable: table, + }) txnID2 := []byte("txn2") _, err := l2.Lock(ctx, table, [][]byte{{3}}, txnID2, pb.LockOptions{ @@ -482,10 +506,23 @@ func TestLockWithBindNotFound(t *testing.T) { // to signal that the caller should retry with the new bind require.True(t, moerr.IsMoErrCode(err, moerr.ErrLockTableBindChanged)) + txnID3 := []byte("txn3") + lockCtx, cancel := context.WithTimeout(ctx, time.Millisecond*200) + defer cancel() + _, err = l2.Lock(lockCtx, table, [][]byte{{3}}, txnID3, pb.LockOptions{ + Granularity: pb.Granularity_Row, + Mode: pb.LockMode_Exclusive, + Policy: pb.WaitPolicy_Wait, + }) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrLockTableBindChanged)) + checkBind( t, - pb.LockTable{ServiceID: l1.serviceID, Version: alloc.version, Table: table, OriginTable: table, Valid: true}, + pb.LockTable{ServiceID: l1.serviceID, Version: alloc.version, Table: table, OriginTable: table, Valid: true, AllocatorID: alloc.allocatorID}, l2) + + require.NoError(t, l2.Unlock(ctx, txnID3, timestamp.Timestamp{})) + require.NoError(t, l2.Unlock(ctx, txnID2, timestamp.Timestamp{})) }, ) } @@ -497,18 +534,24 @@ func TestUnlockWithBindNotFound(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, l3 *service, table uint64) { // change l2's bind to s3, no bind in s3 - l2.handleBindChanged(pb.LockTable{Table: table, ServiceID: "s3", Valid: true, Version: alloc.version}) + l2.handleBindChanged(pb.LockTable{ + Table: table, + ServiceID: l3.serviceID, + Valid: true, + Version: alloc.version, + OriginTable: table, + }) txnID2 := []byte("txn2") l2.Unlock(ctx, txnID2, timestamp.Timestamp{}) checkBind( t, - pb.LockTable{Table: table, ServiceID: "s3", Valid: true, Version: alloc.version}, + pb.LockTable{Table: table, ServiceID: l3.serviceID, Valid: true, Version: alloc.version, OriginTable: table}, l2) }, ) @@ -521,11 +564,17 @@ func TestGetLockWithBindNotFound(t *testing.T) { func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, l3 *service, table uint64) { // change l2's bind to s3, no bind in s3 - l2.handleBindChanged(pb.LockTable{Table: table, ServiceID: "s3", Valid: true, Version: alloc.version}) + l2.handleBindChanged(pb.LockTable{ + Table: table, + ServiceID: l3.serviceID, + Valid: true, + Version: alloc.version, + OriginTable: table, + }) txnID2 := []byte("txn2") lt, err := l2.getLockTable(0, table) @@ -534,7 +583,7 @@ func TestGetLockWithBindNotFound(t *testing.T) { checkBind( t, - pb.LockTable{ServiceID: l1.serviceID, Version: alloc.version, Table: table, OriginTable: table, Valid: true}, + pb.LockTable{ServiceID: l1.serviceID, Version: alloc.version, Table: table, OriginTable: table, Valid: true, AllocatorID: alloc.allocatorID}, l2) }, ) @@ -628,7 +677,7 @@ func runBindChangedTests( fn func( ctx context.Context, alloc *lockTableAllocator, - l1, l2 *service, + l1, l2, l3 *service, table uint64)) { var skip atomic.Bool runLockServiceTestsWithAdjustConfig( @@ -668,7 +717,7 @@ func runBindChangedTests( time.Sleep(time.Millisecond * 100) } - fn(ctx, alloc, l1, l2, table1) + fn(ctx, alloc, l1, l2, s[2], table1) }, func(c *Config) { c.KeepBindDuration.Duration = time.Second diff --git a/pkg/lockservice/service_test.go b/pkg/lockservice/service_test.go index 3d2f204ef5b58..da3ef13e4ee05 100644 --- a/pkg/lockservice/service_test.go +++ b/pkg/lockservice/service_test.go @@ -2105,16 +2105,28 @@ func TestIssue3538(t *testing.T) { require.NoError(t, l1.Unlock(ctx, []byte("txn1"), timestamp.Timestamp{})) + txnID := []byte("txn2") + txnSeq := byte(3) for { _, err = l2.Lock( ctx, 0, [][]byte{{1}}, - []byte("txn2"), + txnID, option) if err == nil { break } + if moerr.IsMoErrCode(err, moerr.ErrLockTableBindChanged) { + txnID = []byte{txnSeq} + txnSeq++ + } else if moerr.IsMoErrCode(err, moerr.ErrRetryForCNRollingRestart) || + moerr.IsMoErrCode(err, moerr.ErrBackendClosed) || + moerr.IsMoErrCode(err, moerr.ErrBackendCannotConnect) { + // retry with the same txn while the old owner is draining + } else { + require.NoError(t, err) + } select { case <-ctx.Done(): panic("timeout bug") @@ -3348,23 +3360,49 @@ func TestAllocatorObserverDoesNotPurgeSameEpochBindVersions(t *testing.T) { ) } -func TestGetBindAllowsRegressedAllocatorVersionWithoutPurging(t *testing.T) { +func TestGetBindPurgesStaleBindWhenAllocatorIDChangesWithRegressedVersion(t *testing.T) { runLockServiceTests( t, []string{"s1"}, func(alloc *lockTableAllocator, s []*service) { l1 := s[0] - table := uint64(19001) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + staleTable := uint64(19001) + freshTable := uint64(19002) + option := newTestRowExclusiveOptions() + + _, err := l1.Lock(ctx, staleTable, newTestRows(1), newTestTxnID(1), option) + require.NoError(t, err) + require.NoError(t, l1.Unlock(ctx, newTestTxnID(1), timestamp.Timestamp{})) + + staleBind := l1.tableGroups.get(0, staleTable).getBind() + oldAllocatorID := staleBind.AllocatorID + require.NotEmpty(t, oldAllocatorID) l1.allocatorVersionMu.Lock() - l1.lastAllocatorVersion = alloc.version + 1 - lastAllocatorVersion := l1.lastAllocatorVersion + l1.lastAllocatorID = oldAllocatorID + l1.lastAllocatorVersion = staleBind.Version l1.allocatorVersionMu.Unlock() - _, err := l1.getLockTableWithCreate(0, table, newTestRows(1), pb.Sharding_None) + alloc.mu.Lock() + alloc.mu.services = make(map[string]*serviceBinds) + alloc.mu.lockTables = make(map[uint32]map[uint64]pb.LockTable) + alloc.allocatorID = "restarted-allocator-with-lower-version" + alloc.version = staleBind.Version - 1 + restartedVersion := alloc.version + restartedAllocatorID := alloc.allocatorID + alloc.mu.Unlock() + + _, err = l1.getLockTableWithCreate(0, freshTable, newTestRows(2), pb.Sharding_None) require.NoError(t, err) - require.Equal(t, lastAllocatorVersion, l1.lastAllocatorVersion) - require.Equal(t, alloc.version, l1.tableGroups.get(0, table).getBind().Version) + require.Nil(t, l1.tableGroups.get(0, staleTable)) + require.Equal(t, restartedVersion, l1.lastAllocatorVersion) + require.Equal(t, restartedAllocatorID, l1.lastAllocatorID) + freshBind := l1.tableGroups.get(0, freshTable).getBind() + require.Equal(t, restartedVersion, freshBind.Version) + require.Equal(t, restartedAllocatorID, freshBind.AllocatorID) }, ) } @@ -3414,9 +3452,11 @@ func TestKeepaliveOKFalseWithNewAllocatorVersionPurgesStaleBinds(t *testing.T) { alloc.mu.Lock() oldVersion := alloc.version + oldAllocatorID := alloc.allocatorID alloc.mu.Unlock() l1.allocatorVersionMu.Lock() + l1.lastAllocatorID = oldAllocatorID l1.lastAllocatorVersion = oldVersion l1.allocatorVersionMu.Unlock() @@ -3432,6 +3472,7 @@ func TestKeepaliveOKFalseWithNewAllocatorVersionPurgesStaleBinds(t *testing.T) { ServiceID: l1.serviceID, Version: oldVersion, Valid: true, + AllocatorID: oldAllocatorID, } l1.tableGroups.set(0, staleLocalTable, l1.createLockTableByBind(staleLocalBind)) @@ -3442,6 +3483,7 @@ func TestKeepaliveOKFalseWithNewAllocatorVersionPurgesStaleBinds(t *testing.T) { ServiceID: "remote-service", Version: oldVersion, Valid: true, + AllocatorID: oldAllocatorID, } l1.tableGroups.set( 0, @@ -3465,7 +3507,9 @@ func TestKeepaliveOKFalseWithNewAllocatorVersionPurgesStaleBinds(t *testing.T) { alloc.mu.Lock() alloc.mu.services = make(map[string]*serviceBinds) alloc.mu.lockTables = make(map[uint32]map[uint64]pb.LockTable) + alloc.allocatorID = "restarted-allocator-for-keepalive" alloc.version++ + restartedAllocatorID := alloc.allocatorID restartedVersion := alloc.version alloc.mu.Unlock() @@ -3473,6 +3517,7 @@ func TestKeepaliveOKFalseWithNewAllocatorVersionPurgesStaleBinds(t *testing.T) { currentProxyBind.Table = currentProxyTable currentProxyBind.OriginTable = currentProxyTable currentProxyBind.Version = restartedVersion + currentProxyBind.AllocatorID = restartedAllocatorID currentProxy := l1.createLockTableByBind(currentProxyBind) _, ok = currentProxy.(*localLockTableProxy) require.True(t, ok) @@ -3493,6 +3538,7 @@ func TestKeepaliveOKFalseWithNewAllocatorVersionPurgesStaleBinds(t *testing.T) { require.NotNil(t, preserved) require.Equal(t, currentProxyBind, preserved.getBind()) require.Equal(t, restartedVersion, l1.lastAllocatorVersion) + require.Equal(t, restartedAllocatorID, l1.lastAllocatorID) }, func(cfg *Config) { cfg.EnableRemoteLocalProxy = true @@ -4618,7 +4664,7 @@ func TestIssue2128(t *testing.T) { [][]byte{{1}}, []byte("txn1"), option) - require.NoError(t, err) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrLockTableBindChanged)) }, ) } diff --git a/pkg/lockservice/txn.go b/pkg/lockservice/txn.go index 9bbe97393175c..c2135787f55d7 100644 --- a/pkg/lockservice/txn.go +++ b/pkg/lockservice/txn.go @@ -35,6 +35,9 @@ var ( type tableLockHolder struct { tableKeys map[uint64]*cowSlice tableBinds map[uint64]pb.LockTable + // tableBindIntents records bind versions touched before a lock attempt + // finishes, so bind-change fencing also covers failed in-flight attempts. + tableBindIntents map[uint64]pb.LockTable } // activeTxn one goroutine write, multi goroutine read @@ -47,6 +50,7 @@ type activeTxn struct { lockHolders map[uint32]*tableLockHolder remoteService string deadlockFound bool + bindChanged bool // test-only hook: called before lockAdded; return non-nil to abort beforeLockAdded func(txnID []byte, locks [][]byte) error @@ -138,6 +142,13 @@ func (txn *activeTxn) lockAdded( return nil } +func (txn *activeTxn) lockTableBindTouched(bind pb.LockTable) { + h := txn.getHoldLocksLocked(bind.Group) + if _, ok := h.tableBindIntents[bind.Table]; !ok { + h.tableBindIntents[bind.Table] = bind + } +} + func (txn *activeTxn) close( txnID []byte, commitTS timestamp.Timestamp, @@ -220,8 +231,13 @@ func (txn *activeTxn) reset() { for table, cs := range h.tableKeys { cs.close() delete(h.tableKeys, table) + } + for table := range h.tableBinds { delete(h.tableBinds, table) } + for table := range h.tableBindIntents { + delete(h.tableBindIntents, table) + } delete(txn.lockHolders, g) } @@ -230,6 +246,7 @@ func (txn *activeTxn) reset() { txn.blockedWaiters = txn.blockedWaiters[:0] txn.remoteService = "" txn.deadlockFound = false + txn.bindChanged = false } func (txn *activeTxn) abort( @@ -257,6 +274,28 @@ func (txn *activeTxn) abort( } } +func (txn *activeTxn) fenceByBindChanged(bind pb.LockTable, logger *log.MOLogger) bool { + txn.Lock() + defer txn.Unlock() + + h, ok := txn.lockHolders[bind.Group] + if !ok { + return false + } + actual, actualOK := h.tableBinds[bind.Table] + intent, intentOK := h.tableBindIntents[bind.Table] + if (!actualOK || !actual.Changed(bind)) && + (!intentOK || !intent.Changed(bind)) { + return false + } + + txn.bindChanged = true + for _, w := range txn.blockedWaiters { + w.notify(notifyValue{err: ErrLockTableBindChanged}, logger) + } + return true +} + func (txn *activeTxn) cancelBlocks( logger *log.MOLogger, ) { @@ -423,8 +462,9 @@ func (txn *activeTxn) getHoldLocksLocked(group uint32) *tableLockHolder { return h } h = &tableLockHolder{ - tableKeys: make(map[uint64]*cowSlice), - tableBinds: make(map[uint64]pb.LockTable), + tableKeys: make(map[uint64]*cowSlice), + tableBinds: make(map[uint64]pb.LockTable), + tableBindIntents: make(map[uint64]pb.LockTable), } txn.lockHolders[group] = h return h diff --git a/pkg/lockservice/types.go b/pkg/lockservice/types.go index 738c93171bbfd..83382f0ce242b 100644 --- a/pkg/lockservice/types.go +++ b/pkg/lockservice/types.go @@ -161,7 +161,7 @@ type lockTable interface { // getBind returns lock table binding getBind() pb.LockTable // close close the locktable - close() + close(...closeReason) } // LockTableAllocator is used to managing the binding relationship between diff --git a/pkg/pb/lock/lock.go b/pkg/pb/lock/lock.go index 12e35c0551f20..37aec878cb330 100644 --- a/pkg/pb/lock/lock.go +++ b/pkg/pb/lock/lock.go @@ -105,7 +105,8 @@ func (m *Response) DebugString() string { // Changed returns true if LockTable bind changed func (m LockTable) Changed(v LockTable) bool { return m.Version != v.Version || - m.ServiceID != v.ServiceID + m.ServiceID != v.ServiceID || + (m.AllocatorID != "" && v.AllocatorID != "" && m.AllocatorID != v.AllocatorID) } // Equal return true means same bind @@ -115,7 +116,10 @@ func (m LockTable) Equal(v LockTable) bool { // DebugString returns the debug string func (m LockTable) DebugString() string { - return fmt.Sprintf("%d-%d(%d)-%s-%d", m.Group, m.Table, m.OriginTable, m.ServiceID, m.Version) + if m.AllocatorID == "" { + return fmt.Sprintf("%d-%d(%d)-%s-%d", m.Group, m.Table, m.OriginTable, m.ServiceID, m.Version) + } + return fmt.Sprintf("%d-%d(%d)-%s-%d-%s", m.Group, m.Table, m.OriginTable, m.ServiceID, m.Version, m.AllocatorID) } // WithGranularity set rows granularity, the default granularity is Row. diff --git a/pkg/pb/lock/lock.pb.go b/pkg/pb/lock/lock.pb.go index fb342736ca2ca..2672635b676bb 100644 --- a/pkg/pb/lock/lock.pb.go +++ b/pkg/pb/lock/lock.pb.go @@ -382,10 +382,12 @@ type LockTable struct { // Version will incr if CN-Table bind changed. Version uint64 `protobuf:"varint,3,opt,name=Version,proto3" json:"Version,omitempty"` // Valid false if the service is disabled, and no new service bind this table - Valid bool `protobuf:"varint,4,opt,name=Valid,proto3" json:"Valid,omitempty"` - Group uint32 `protobuf:"varint,5,opt,name=Group,proto3" json:"Group,omitempty"` - Sharding Sharding `protobuf:"varint,6,opt,name=Sharding,proto3,enum=lock.Sharding" json:"Sharding,omitempty"` - OriginTable uint64 `protobuf:"varint,7,opt,name=OriginTable,proto3" json:"OriginTable,omitempty"` + Valid bool `protobuf:"varint,4,opt,name=Valid,proto3" json:"Valid,omitempty"` + Group uint32 `protobuf:"varint,5,opt,name=Group,proto3" json:"Group,omitempty"` + Sharding Sharding `protobuf:"varint,6,opt,name=Sharding,proto3,enum=lock.Sharding" json:"Sharding,omitempty"` + OriginTable uint64 `protobuf:"varint,7,opt,name=OriginTable,proto3" json:"OriginTable,omitempty"` + // AllocatorID identifies the allocator process that created this bind. + AllocatorID string `protobuf:"bytes,8,opt,name=AllocatorID,proto3" json:"AllocatorID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -473,6 +475,13 @@ func (m *LockTable) GetOriginTable() uint64 { return 0 } +func (m *LockTable) GetAllocatorID() string { + if m != nil { + return m.AllocatorID + } + return "" +} + // Request is used to send a request for a LockTable related operation to another // service. type Request struct { @@ -1455,6 +1464,7 @@ func (m *GetBindRequest) GetOriginTable() uint64 { type GetBindResponse struct { LockTable LockTable `protobuf:"bytes,2,opt,name=LockTable,proto3" json:"LockTable"` AllocatorVersion uint64 `protobuf:"varint,3,opt,name=AllocatorVersion,proto3" json:"AllocatorVersion,omitempty"` + AllocatorID string `protobuf:"bytes,4,opt,name=AllocatorID,proto3" json:"AllocatorID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1507,6 +1517,13 @@ func (m *GetBindResponse) GetAllocatorVersion() uint64 { return 0 } +func (m *GetBindResponse) GetAllocatorID() string { + if m != nil { + return m.AllocatorID + } + return "" +} + // KeepLockTableBindRequest keep lock table bind request. CN -> TN type KeepLockTableBindRequest struct { ServiceID string `protobuf:"bytes,1,opt,name=ServiceID,proto3" json:"ServiceID,omitempty"` @@ -1584,6 +1601,7 @@ type KeepLockTableBindResponse struct { OK bool `protobuf:"varint,1,opt,name=OK,proto3" json:"OK,omitempty"` Status Status `protobuf:"varint,2,opt,name=Status,proto3,enum=lock.Status" json:"Status,omitempty"` AllocatorVersion uint64 `protobuf:"varint,3,opt,name=AllocatorVersion,proto3" json:"AllocatorVersion,omitempty"` + AllocatorID string `protobuf:"bytes,4,opt,name=AllocatorID,proto3" json:"AllocatorID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1643,6 +1661,13 @@ func (m *KeepLockTableBindResponse) GetAllocatorVersion() uint64 { return 0 } +func (m *KeepLockTableBindResponse) GetAllocatorID() string { + if m != nil { + return m.AllocatorID + } + return "" +} + // SetRestartServiceRequest set restart lock service request from allocator request. operator -> TN type SetRestartServiceRequest struct { ServiceID string `protobuf:"bytes,1,opt,name=ServiceID,proto3" json:"ServiceID,omitempty"` @@ -2905,141 +2930,142 @@ func init() { func init() { proto.RegisterFile("lock.proto", fileDescriptor_164ad2988c7acaf1) } var fileDescriptor_164ad2988c7acaf1 = []byte{ - // 2130 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x59, 0xcd, 0x73, 0xdb, 0xc6, - 0x15, 0x17, 0x48, 0x8a, 0x1f, 0x8f, 0xa4, 0x04, 0xad, 0x64, 0x19, 0x52, 0x13, 0x99, 0x45, 0x95, + // 2153 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x59, 0x5f, 0x73, 0xdb, 0xc6, + 0x11, 0x17, 0x48, 0x8a, 0x7f, 0x96, 0xa4, 0x04, 0x9d, 0x64, 0x19, 0x52, 0x13, 0x99, 0x45, 0x95, 0x19, 0x46, 0x69, 0xac, 0x5a, 0xae, 0x93, 0xd4, 0x69, 0x3d, 0x95, 0x29, 0x5b, 0x51, 0x6c, 0x59, - 0x99, 0x25, 0xed, 0xce, 0xf4, 0x06, 0x91, 0x6b, 0x09, 0x23, 0x0a, 0x60, 0x41, 0x50, 0xa2, 0x0e, - 0x3d, 0x76, 0xa6, 0x33, 0xbd, 0xf7, 0xd8, 0xe9, 0x31, 0x97, 0xfe, 0x1f, 0x39, 0x66, 0xa6, 0xf7, - 0x7e, 0xb8, 0xff, 0x48, 0xe7, 0xed, 0x2e, 0x88, 0x5d, 0x7c, 0x88, 0x71, 0x6e, 0xd8, 0xf7, 0xde, - 0xfe, 0xde, 0xee, 0xdb, 0x87, 0xdf, 0xbe, 0x07, 0x00, 0x0c, 0xfd, 0xfe, 0xc5, 0xfd, 0x51, 0xe0, - 0x87, 0x3e, 0x29, 0xe1, 0xf3, 0xe6, 0xa7, 0x67, 0x6e, 0x78, 0x3e, 0x39, 0xbd, 0xdf, 0xf7, 0x2f, - 0x77, 0xcf, 0xfc, 0x33, 0x7f, 0x97, 0x2b, 0x4f, 0x27, 0x6f, 0xf9, 0x88, 0x0f, 0xf8, 0x93, 0x98, - 0xb4, 0xb9, 0x1c, 0xba, 0x97, 0x6c, 0x1c, 0x3a, 0x97, 0x23, 0x21, 0xb0, 0xff, 0x56, 0x84, 0xfa, - 0x4b, 0xbf, 0x7f, 0x71, 0x32, 0x0a, 0x5d, 0xdf, 0x1b, 0x93, 0x87, 0x50, 0x3f, 0x0c, 0x1c, 0x6f, - 0x32, 0x74, 0x02, 0x37, 0xbc, 0xb1, 0x8c, 0x96, 0xd1, 0x5e, 0xda, 0x5b, 0xb9, 0xcf, 0xfd, 0x2a, - 0x0a, 0xaa, 0x5a, 0x11, 0x1b, 0x4a, 0xc7, 0xfe, 0x80, 0x59, 0x05, 0x6e, 0xbd, 0x24, 0xac, 0x11, - 0x15, 0xa5, 0x94, 0xeb, 0x48, 0x1b, 0xca, 0xdf, 0xf8, 0x43, 0xb7, 0x7f, 0x63, 0x15, 0xb9, 0x95, - 0x29, 0xac, 0x7e, 0xe7, 0xb8, 0xa1, 0x90, 0x53, 0xa9, 0x27, 0x1f, 0x40, 0xed, 0xb9, 0x1f, 0x5c, - 0x3b, 0xc1, 0xa0, 0xe7, 0x5b, 0xa5, 0x96, 0xd1, 0xae, 0xd1, 0x58, 0x40, 0xda, 0xb0, 0xdc, 0x73, - 0x4e, 0x87, 0xec, 0x80, 0xbd, 0xed, 0x9c, 0x3b, 0xde, 0x19, 0x1b, 0x58, 0x8b, 0x2d, 0xa3, 0x5d, - 0xa5, 0x49, 0x31, 0xe2, 0x50, 0x16, 0x06, 0x37, 0xe8, 0xc2, 0x2a, 0xb7, 0x8c, 0x76, 0x91, 0xc6, - 0x02, 0xb2, 0x06, 0x8b, 0x87, 0x81, 0x3f, 0x19, 0x59, 0x95, 0x96, 0xd1, 0x6e, 0x52, 0x31, 0x20, - 0x3b, 0x50, 0xed, 0x9e, 0x3b, 0xc1, 0xc0, 0xf5, 0xce, 0xac, 0xaa, 0xba, 0x9b, 0x48, 0x4a, 0x67, - 0x7a, 0xf2, 0x18, 0xa0, 0xeb, 0x39, 0xa3, 0xee, 0xb9, 0x1f, 0xf6, 0xc6, 0x56, 0xad, 0x65, 0xb4, - 0xeb, 0x7b, 0x6b, 0xf7, 0xe3, 0x00, 0xf7, 0xa2, 0xa7, 0xa7, 0xa5, 0xef, 0xfe, 0x75, 0x6f, 0x81, - 0x2a, 0xd6, 0xb8, 0x0b, 0x8c, 0x0f, 0xae, 0x04, 0xcd, 0xfc, 0x49, 0x68, 0x01, 0x5f, 0x61, 0x52, - 0x6c, 0xff, 0xd3, 0x80, 0x1a, 0xca, 0xf8, 0xee, 0x70, 0xd5, 0xfc, 0x81, 0x1f, 0x4c, 0x89, 0x8a, - 0x01, 0xee, 0xb4, 0xcb, 0x82, 0x2b, 0xb7, 0xcf, 0x8e, 0x0e, 0xf8, 0x21, 0xd4, 0x68, 0x2c, 0x20, - 0x16, 0x54, 0xde, 0xb0, 0x60, 0xec, 0xfa, 0x1e, 0x0f, 0x7d, 0x89, 0x46, 0x43, 0x44, 0x7b, 0xe3, - 0x0c, 0xdd, 0x01, 0x8f, 0x72, 0x95, 0x8a, 0x41, 0x1c, 0x99, 0xc5, 0xbc, 0xc8, 0x94, 0xe7, 0x44, - 0xa6, 0x05, 0xf5, 0x93, 0xc0, 0x3d, 0x73, 0x3d, 0xb1, 0xd6, 0x0a, 0xf7, 0xaa, 0x8a, 0xec, 0xbf, - 0x00, 0x54, 0x28, 0xfb, 0xc3, 0x84, 0x8d, 0x43, 0x71, 0x4e, 0xfc, 0xf1, 0xe8, 0x40, 0xee, 0x2b, - 0x16, 0x90, 0x87, 0xca, 0xf6, 0xf9, 0xde, 0xea, 0x7b, 0xcb, 0x71, 0x82, 0x71, 0xb1, 0x8c, 0xaf, - 0x12, 0xa6, 0x6d, 0x28, 0x1f, 0xb3, 0xf0, 0xdc, 0x1f, 0xc8, 0x64, 0x6b, 0x88, 0x19, 0x42, 0x46, - 0xa5, 0x8e, 0x7c, 0x02, 0x25, 0x9c, 0xc2, 0x77, 0x5f, 0x8f, 0x92, 0x1c, 0x25, 0xd2, 0xbb, 0xc4, - 0xe5, 0x46, 0xe4, 0x01, 0x94, 0x5f, 0x7b, 0x68, 0xc1, 0xc3, 0x52, 0xdf, 0x5b, 0x15, 0xe6, 0x42, - 0xa6, 0x4f, 0x90, 0x86, 0xe4, 0x37, 0x00, 0x87, 0x2c, 0xec, 0x4d, 0x3d, 0xee, 0xa5, 0xcc, 0xa7, - 0xdd, 0x95, 0xaf, 0xd2, 0x4c, 0xae, 0x4f, 0x55, 0x26, 0x90, 0x23, 0x58, 0x3a, 0x64, 0x21, 0xe6, - 0x82, 0xeb, 0x9d, 0xbd, 0x74, 0xc7, 0x21, 0x0f, 0x64, 0x7d, 0xef, 0x27, 0x33, 0x08, 0x45, 0xa7, - 0xc3, 0x24, 0x26, 0x92, 0x5f, 0x42, 0xe5, 0x90, 0x85, 0x4f, 0x5d, 0x6f, 0xc0, 0xb3, 0x1a, 0xf3, - 0x34, 0xc2, 0x40, 0xa1, 0x3e, 0x39, 0x32, 0x25, 0x14, 0x56, 0x5e, 0x30, 0x36, 0x8a, 0xe3, 0x8c, - 0xf3, 0x45, 0x9e, 0x6f, 0x89, 0xf9, 0x29, 0xb5, 0x8e, 0x94, 0x9e, 0x8e, 0x9b, 0x42, 0x21, 0x65, - 0x97, 0x7e, 0xc8, 0x78, 0x5c, 0x40, 0xdd, 0x94, 0xae, 0x4b, 0x6c, 0x4a, 0x57, 0x92, 0x97, 0xb0, - 0xcc, 0x13, 0xd6, 0x09, 0x99, 0x4c, 0x76, 0xab, 0xce, 0xb1, 0x3e, 0x10, 0x58, 0x09, 0xa5, 0x0e, - 0x96, 0x9c, 0x4a, 0x3a, 0xd0, 0xe8, 0x38, 0x9e, 0xe7, 0x87, 0x1d, 0xff, 0xf2, 0xd2, 0x0d, 0xad, - 0x06, 0x87, 0xda, 0x10, 0x50, 0xaa, 0x46, 0xc7, 0xd1, 0x26, 0x21, 0xc8, 0x21, 0x0b, 0xf7, 0xfb, - 0xa1, 0x7b, 0xc5, 0x7a, 0x53, 0xcf, 0x6a, 0xaa, 0x20, 0xaa, 0x26, 0x01, 0xa2, 0xaa, 0x30, 0xec, - 0x5d, 0x16, 0x52, 0xe4, 0x8e, 0x20, 0x8c, 0x76, 0xb6, 0xa4, 0x86, 0x3d, 0xa5, 0x4e, 0x84, 0x3d, - 0xa5, 0x47, 0xcc, 0x8e, 0xe3, 0x25, 0x30, 0x97, 0x55, 0xcc, 0x94, 0x3a, 0x81, 0x99, 0xd2, 0x93, - 0xd7, 0x40, 0x28, 0xbb, 0x74, 0x5c, 0xaf, 0x37, 0xf5, 0x8e, 0xbc, 0x08, 0xd4, 0xe4, 0xa0, 0xf7, - 0x04, 0x68, 0x5a, 0xaf, 0xa3, 0x66, 0x00, 0x90, 0xdf, 0x42, 0xbd, 0x73, 0xce, 0xfa, 0x17, 0x27, - 0xc1, 0xe8, 0xdc, 0xf1, 0xac, 0x15, 0x8e, 0x67, 0xc9, 0x45, 0xc6, 0x0a, 0x1d, 0x48, 0x9d, 0x82, - 0x89, 0x41, 0xd9, 0x78, 0x72, 0xc9, 0x8e, 0xbc, 0x2b, 0x3c, 0xe5, 0xce, 0x2b, 0x8b, 0xa8, 0x89, - 0x91, 0x50, 0x26, 0x12, 0x23, 0xa1, 0x25, 0x0e, 0xac, 0xef, 0x9f, 0xfa, 0x41, 0x28, 0x32, 0xef, - 0x80, 0x39, 0x03, 0x04, 0xc1, 0xd3, 0x5d, 0xe3, 0xa0, 0x3f, 0x13, 0xa0, 0xd9, 0x36, 0x3a, 0x76, - 0x0e, 0x90, 0xfd, 0x2d, 0x40, 0x95, 0xb2, 0xf1, 0xc8, 0xf7, 0xc6, 0x6c, 0x0e, 0x1d, 0xc6, 0xcc, - 0x56, 0xb8, 0x85, 0xd9, 0xd6, 0x60, 0xf1, 0x59, 0x10, 0xf8, 0x01, 0xa7, 0xbf, 0x06, 0x15, 0x03, - 0xf2, 0x31, 0x54, 0x5e, 0xb1, 0x6b, 0xfe, 0x16, 0x97, 0x32, 0x89, 0x94, 0x46, 0x7a, 0xf2, 0x73, - 0x49, 0x8d, 0x82, 0xeb, 0x88, 0x4a, 0x8d, 0x62, 0x99, 0x1a, 0x37, 0xee, 0xcd, 0xb8, 0xb1, 0xac, - 0xb2, 0x4b, 0xc4, 0x8d, 0xda, 0x8c, 0x88, 0x1c, 0x9f, 0x68, 0xe4, 0x58, 0x51, 0x4f, 0x59, 0x25, - 0x47, 0x6d, 0xae, 0xca, 0x8e, 0x5f, 0xa7, 0xd8, 0xb1, 0xaa, 0x9e, 0x71, 0x92, 0x1d, 0x35, 0x9c, - 0x24, 0x3d, 0x3e, 0x8a, 0xe9, 0x51, 0xd0, 0xdb, 0x9d, 0x04, 0x3d, 0x6a, 0xb3, 0x67, 0xfc, 0xd8, - 0xcd, 0xe2, 0x47, 0x50, 0xf3, 0x3f, 0x83, 0x1f, 0x35, 0xa8, 0x0c, 0x82, 0xfc, 0x3a, 0x45, 0x90, - 0x1a, 0xa9, 0x25, 0x09, 0x52, 0xdf, 0x57, 0x82, 0x21, 0x8f, 0xd3, 0x0c, 0x29, 0x68, 0xed, 0xc3, - 0x1c, 0x86, 0xd4, 0xd0, 0x52, 0x14, 0x79, 0x90, 0xa0, 0x48, 0xc1, 0x6e, 0x9b, 0x59, 0x14, 0xa9, - 0x01, 0xe9, 0x1c, 0x79, 0x90, 0xe0, 0xc8, 0x25, 0x15, 0x45, 0xe7, 0x48, 0x1d, 0x45, 0x23, 0xc9, - 0x6e, 0x16, 0x49, 0x2e, 0xab, 0xb1, 0xcf, 0x20, 0x49, 0x3d, 0xf6, 0x69, 0x96, 0xec, 0x66, 0xb1, - 0xa4, 0x46, 0x68, 0x19, 0x2c, 0xa9, 0x83, 0xa6, 0x69, 0xf2, 0x4d, 0x26, 0x4d, 0x0a, 0x5a, 0x6b, - 0xe5, 0xd3, 0xa4, 0x06, 0x9b, 0xc5, 0x93, 0xfb, 0x3a, 0x4f, 0x12, 0xed, 0xbe, 0x52, 0x79, 0x52, - 0x43, 0xd2, 0x88, 0xf2, 0x38, 0x4d, 0x94, 0xab, 0x6a, 0x7e, 0xa4, 0x88, 0x52, 0xcf, 0x8f, 0x24, - 0x53, 0x9e, 0xce, 0x61, 0xca, 0xed, 0xdb, 0x99, 0x52, 0x03, 0xcf, 0xa3, 0xca, 0x3f, 0x1b, 0xa2, - 0x5f, 0x89, 0x8a, 0x47, 0x2c, 0x88, 0xa7, 0x9e, 0x64, 0xca, 0x06, 0x15, 0x83, 0x39, 0x05, 0x31, - 0x81, 0x12, 0xf5, 0xaf, 0xc7, 0x56, 0xb1, 0x55, 0x6c, 0x37, 0x28, 0x7f, 0x26, 0x0f, 0xa0, 0x22, - 0x5b, 0xa0, 0x74, 0x39, 0x28, 0x15, 0xd1, 0xeb, 0x2f, 0x87, 0xf6, 0x63, 0x68, 0xa8, 0xef, 0x20, - 0xd9, 0x81, 0x32, 0x46, 0x64, 0x18, 0xf2, 0xb5, 0xd4, 0x23, 0x6a, 0x16, 0xb2, 0x88, 0xfd, 0xc4, - 0xc8, 0xfe, 0x12, 0x56, 0x52, 0x25, 0x60, 0xce, 0x5e, 0x4c, 0x28, 0x52, 0xff, 0x9a, 0xef, 0xa2, - 0x41, 0xf1, 0xd1, 0x76, 0x80, 0xa4, 0x29, 0x52, 0x16, 0xf3, 0x13, 0xd1, 0x1a, 0x2c, 0x52, 0x31, - 0x20, 0x8f, 0xa0, 0xae, 0x72, 0x64, 0xa1, 0x55, 0x6c, 0xd7, 0xf7, 0x9a, 0x71, 0xef, 0xd5, 0x9b, - 0x7a, 0x51, 0x66, 0x28, 0x76, 0xf6, 0x13, 0xb8, 0x93, 0x59, 0x5f, 0x92, 0x8f, 0xa0, 0x88, 0x07, - 0x2a, 0x76, 0x98, 0x89, 0x83, 0x7a, 0xfb, 0x04, 0xd6, 0xb3, 0x19, 0x38, 0xb9, 0x20, 0xe3, 0x07, - 0x2e, 0xa8, 0x0f, 0x15, 0xa9, 0xcd, 0x3f, 0xf2, 0x4e, 0xc0, 0x9c, 0x90, 0x0d, 0x4e, 0xbc, 0xe8, - 0xc8, 0x67, 0x02, 0xb2, 0x0d, 0x4d, 0x9c, 0xce, 0x82, 0xfd, 0xc1, 0x20, 0x60, 0xe3, 0x31, 0xbf, - 0x18, 0x6b, 0x54, 0x17, 0xda, 0x7f, 0x35, 0xa0, 0xa9, 0x15, 0xf4, 0x39, 0xbe, 0x3e, 0x83, 0xaa, - 0x20, 0xb3, 0x5e, 0x57, 0xb6, 0x24, 0xb7, 0xf5, 0x7d, 0x33, 0x5b, 0xf2, 0x39, 0xd4, 0x8e, 0x27, - 0xa1, 0x23, 0xd2, 0xac, 0xc8, 0x77, 0x2e, 0xdb, 0x88, 0x67, 0xd3, 0x30, 0x70, 0x22, 0x5d, 0xd4, - 0xcf, 0xcc, 0x6c, 0x6d, 0x13, 0x96, 0xf4, 0xcb, 0xd4, 0xfe, 0xd6, 0xe0, 0xf7, 0x9f, 0x52, 0x73, - 0xeb, 0x49, 0x6f, 0x24, 0x93, 0x7e, 0xd6, 0x39, 0x16, 0xd4, 0xce, 0x71, 0xd6, 0xeb, 0x15, 0xf3, - 0x7a, 0xbd, 0xd2, 0xfb, 0xf5, 0x7a, 0x8b, 0xe9, 0x5e, 0x2f, 0x80, 0xe5, 0xc4, 0x45, 0xfa, 0xe3, - 0x9a, 0xba, 0x1d, 0x30, 0xf7, 0x87, 0x43, 0xbf, 0xef, 0x84, 0x7e, 0xa0, 0x37, 0xb4, 0x29, 0xb9, - 0xfd, 0x0f, 0x03, 0xac, 0xbc, 0xe6, 0x64, 0x4e, 0xa0, 0xb6, 0xa1, 0xdc, 0x0d, 0x9d, 0x70, 0x32, - 0xd6, 0x2b, 0x2c, 0x21, 0xa3, 0x52, 0x47, 0xd6, 0xa1, 0xcc, 0x73, 0x21, 0x62, 0x11, 0x39, 0x22, - 0x8f, 0x00, 0x66, 0x3e, 0x91, 0x4a, 0x8a, 0xf9, 0x5b, 0x53, 0x0c, 0xed, 0x3f, 0xc2, 0x46, 0x6e, - 0xad, 0x40, 0x96, 0xa0, 0x70, 0xf2, 0x82, 0x2f, 0xb4, 0x4a, 0x0b, 0x27, 0x2f, 0x7e, 0xe0, 0x0a, - 0xdf, 0x27, 0x5c, 0x5f, 0x80, 0x95, 0xd7, 0x53, 0xdc, 0x1e, 0x2d, 0xfb, 0x13, 0xd8, 0xc8, 0xbd, - 0x68, 0x93, 0x0b, 0x47, 0x37, 0x79, 0x6d, 0xc6, 0x7c, 0x37, 0xb9, 0x57, 0x6f, 0xca, 0xcd, 0xaf, - 0x60, 0x23, 0xb7, 0xf1, 0x98, 0xe3, 0xe7, 0x31, 0x6c, 0xe6, 0x5f, 0xc6, 0xa2, 0x34, 0x97, 0x5a, - 0x49, 0xb3, 0xb1, 0xc0, 0x7e, 0x04, 0x77, 0x32, 0xdb, 0xd7, 0x39, 0x2e, 0xdb, 0xb0, 0x9e, 0x5d, - 0xd4, 0xa5, 0xf6, 0xf5, 0x19, 0xac, 0x67, 0xf7, 0xb4, 0x73, 0x3c, 0x7c, 0x0c, 0x77, 0x73, 0x2a, - 0xbd, 0x94, 0x8b, 0xe7, 0xf0, 0xe1, 0xad, 0x8d, 0x0c, 0xf2, 0x7f, 0x38, 0x87, 0xff, 0xc3, 0xa9, - 0x67, 0xff, 0x02, 0xb6, 0x6e, 0xbf, 0xe6, 0x53, 0x9e, 0x29, 0xac, 0x66, 0x74, 0xd9, 0xe4, 0x4b, - 0x68, 0x8a, 0x62, 0x05, 0xaf, 0xbb, 0xf8, 0xc2, 0x90, 0xaf, 0xd4, 0x4c, 0x25, 0x7d, 0xeb, 0xb6, - 0xf6, 0xaf, 0x61, 0x2d, 0xab, 0x2c, 0xc5, 0xdb, 0x40, 0x48, 0xf0, 0x7a, 0x11, 0x67, 0x89, 0xef, - 0xb0, 0x2e, 0xb4, 0x1f, 0xc2, 0x6a, 0x46, 0xcb, 0x3e, 0x27, 0xd6, 0x4f, 0x60, 0x2d, 0xab, 0x86, - 0x8d, 0x3f, 0xb5, 0x19, 0xea, 0xa7, 0x36, 0x53, 0xdc, 0xa6, 0x05, 0xee, 0x9e, 0x5f, 0x9c, 0x07, - 0x40, 0xd2, 0x4d, 0xee, 0x1c, 0xc6, 0x9a, 0xa1, 0x18, 0x11, 0xca, 0xa7, 0xb0, 0x9a, 0x51, 0x02, - 0x22, 0x69, 0xc9, 0x6a, 0x51, 0xac, 0x42, 0x8e, 0xec, 0xcf, 0xa1, 0x36, 0x0b, 0x1c, 0xb1, 0xa0, - 0x12, 0x15, 0xa9, 0xc2, 0x53, 0x34, 0xcc, 0x58, 0xed, 0x9f, 0x8a, 0x51, 0xcd, 0x43, 0x1e, 0x40, - 0x15, 0x93, 0x97, 0x5f, 0xbf, 0xc6, 0x6d, 0x8c, 0x3e, 0x33, 0xc3, 0xab, 0xe3, 0x2b, 0x67, 0xdc, - 0xf1, 0xbd, 0xb7, 0x43, 0xb7, 0x1f, 0xf2, 0xf5, 0x57, 0xa9, 0x2a, 0xc2, 0x83, 0xfa, 0xca, 0x19, - 0x7f, 0x13, 0xb0, 0x2b, 0xd9, 0x72, 0x14, 0xb9, 0x8d, 0x2e, 0x24, 0x5f, 0x40, 0x6d, 0x76, 0xe7, - 0xca, 0xea, 0xed, 0xb6, 0xfb, 0x38, 0x36, 0x7e, 0x8f, 0x8f, 0xc9, 0x2d, 0xa8, 0x47, 0xab, 0x7a, - 0xc1, 0x6e, 0x78, 0x9f, 0xdb, 0xa0, 0xaa, 0x48, 0xb5, 0xc0, 0x28, 0x55, 0x74, 0x0b, 0x8c, 0xec, - 0x16, 0x00, 0xae, 0x5a, 0xd4, 0x1c, 0xbc, 0x5d, 0x6d, 0x50, 0x45, 0x82, 0x91, 0x17, 0x4f, 0xe2, - 0x6b, 0x72, 0x93, 0x46, 0x43, 0x9c, 0xf9, 0x8a, 0x5d, 0x63, 0xe0, 0xf6, 0x07, 0xa2, 0xc5, 0xac, - 0x52, 0x45, 0x62, 0x77, 0xa1, 0xa9, 0x55, 0x10, 0x78, 0x54, 0x17, 0xec, 0x46, 0x56, 0x2d, 0xf8, - 0x88, 0x45, 0xef, 0xf8, 0xc2, 0x1d, 0xc9, 0x28, 0xf3, 0x67, 0x4c, 0xab, 0x80, 0x8d, 0x86, 0x4e, - 0x9f, 0xf5, 0x7c, 0xf9, 0xa9, 0x20, 0x16, 0x20, 0xdd, 0x64, 0x7f, 0x29, 0x99, 0xf3, 0x0a, 0x6c, - 0xc0, 0xdd, 0x9c, 0xc6, 0x61, 0xe7, 0xa7, 0xda, 0xdf, 0x05, 0x52, 0xe1, 0xa5, 0xad, 0xb9, 0x40, - 0x6a, 0xb0, 0x48, 0x31, 0xce, 0xa6, 0xb1, 0xf3, 0x91, 0xc8, 0x23, 0xfe, 0xcf, 0xa0, 0x09, 0xb5, - 0x67, 0xd3, 0xfe, 0x70, 0x32, 0x76, 0xaf, 0x98, 0xb9, 0x40, 0x00, 0xca, 0x58, 0x76, 0xb0, 0x81, - 0x69, 0xec, 0x6c, 0x03, 0xc4, 0xbf, 0x0e, 0x48, 0x15, 0x4a, 0x38, 0x32, 0x17, 0x48, 0x03, 0xaa, - 0xcf, 0x9d, 0x71, 0xf8, 0xdc, 0x71, 0x87, 0xa6, 0xb1, 0x73, 0x2f, 0x2e, 0x64, 0xd0, 0xe6, 0x95, - 0xef, 0x31, 0xe1, 0xed, 0xe9, 0x0d, 0x3a, 0x36, 0x76, 0xfe, 0x5d, 0x88, 0xbe, 0xa7, 0xa0, 0x1e, - 0x1d, 0x0b, 0x3f, 0xa2, 0xda, 0x32, 0x0d, 0xb2, 0xa4, 0x7e, 0xa6, 0x30, 0x0b, 0x84, 0x24, 0x3f, - 0x3b, 0x98, 0x45, 0x94, 0xe9, 0x0c, 0x6e, 0x96, 0x48, 0x7d, 0xf6, 0x49, 0xc1, 0x5c, 0x24, 0x77, - 0x32, 0x3e, 0x14, 0x98, 0x65, 0xb2, 0x0c, 0x75, 0xf9, 0x5f, 0x83, 0x4f, 0xaa, 0x90, 0x15, 0x68, - 0x4a, 0x81, 0xf4, 0x5f, 0x25, 0xab, 0xa9, 0x16, 0xde, 0xac, 0x11, 0x53, 0x6f, 0xc4, 0x4d, 0x40, - 0x89, 0x4a, 0x3b, 0x66, 0x1d, 0x7d, 0xa6, 0x2e, 0x66, 0xb3, 0x41, 0xd6, 0xb3, 0xba, 0x51, 0xb3, - 0x89, 0xe6, 0xa9, 0x0b, 0xd6, 0x5c, 0xc2, 0x25, 0x2a, 0x44, 0x62, 0x2e, 0xe3, 0x7a, 0x12, 0x87, - 0x6b, 0x9a, 0x64, 0x33, 0xaf, 0xf1, 0x33, 0x57, 0x76, 0x58, 0x54, 0xac, 0x88, 0x15, 0x71, 0x60, - 0xdc, 0xee, 0x33, 0x0f, 0x23, 0x61, 0x2e, 0xe0, 0x8a, 0x14, 0xb1, 0x8c, 0xac, 0x69, 0x28, 0xe6, - 0xaf, 0x79, 0xf0, 0xbb, 0x93, 0x7e, 0xdf, 0x2c, 0x28, 0xe2, 0x78, 0xbd, 0x66, 0xf1, 0x69, 0xe7, - 0xfb, 0xff, 0x6e, 0x19, 0xdf, 0xbd, 0xdb, 0x32, 0xbe, 0x7f, 0xb7, 0x65, 0xfc, 0xe7, 0xdd, 0xd6, - 0xc2, 0xdf, 0xff, 0xb7, 0x65, 0xfc, 0x5e, 0xfd, 0x3b, 0x76, 0xe9, 0x84, 0x81, 0x3b, 0xf5, 0x79, - 0x61, 0x1a, 0x0d, 0x3c, 0xb6, 0x3b, 0xba, 0x38, 0xdb, 0x1d, 0x9d, 0xee, 0xe2, 0x82, 0x4f, 0xcb, - 0xfc, 0x9f, 0xd8, 0xc3, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, 0xb0, 0xe6, 0x63, 0xfb, 0x67, 0x1b, - 0x00, 0x00, + 0x99, 0x23, 0xed, 0xce, 0xf4, 0x0d, 0x22, 0xcf, 0x12, 0x46, 0x14, 0xc0, 0x82, 0xa0, 0x44, 0x7d, + 0x80, 0xce, 0x74, 0xa6, 0xaf, 0x9d, 0x3e, 0xb6, 0x7d, 0xcc, 0x4b, 0xbf, 0x47, 0x1e, 0xf3, 0x09, + 0xfa, 0xc7, 0xfd, 0x0c, 0x7d, 0xef, 0xec, 0xdd, 0x81, 0xb8, 0xc3, 0x1f, 0x31, 0xe9, 0xe4, 0x0d, + 0xb7, 0xbb, 0xf7, 0xdb, 0xbb, 0xbd, 0xc3, 0x6f, 0x77, 0x01, 0x80, 0xa1, 0xdf, 0xbf, 0xb8, 0x3f, + 0x0a, 0xfc, 0xd0, 0x27, 0x25, 0x7c, 0xde, 0xfc, 0xf8, 0xcc, 0x0d, 0xcf, 0x27, 0xa7, 0xf7, 0xfb, + 0xfe, 0xe5, 0xee, 0x99, 0x7f, 0xe6, 0xef, 0x72, 0xe5, 0xe9, 0xe4, 0x2d, 0x1f, 0xf1, 0x01, 0x7f, + 0x12, 0x93, 0x36, 0x97, 0x43, 0xf7, 0x92, 0x8d, 0x43, 0xe7, 0x72, 0x24, 0x04, 0xf6, 0x5f, 0x8a, + 0x50, 0x7f, 0xe9, 0xf7, 0x2f, 0x4e, 0x46, 0xa1, 0xeb, 0x7b, 0x63, 0xf2, 0x10, 0xea, 0x87, 0x81, + 0xe3, 0x4d, 0x86, 0x4e, 0xe0, 0x86, 0x37, 0x96, 0xd1, 0x32, 0xda, 0x4b, 0x7b, 0x2b, 0xf7, 0xb9, + 0x5f, 0x45, 0x41, 0x55, 0x2b, 0x62, 0x43, 0xe9, 0xd8, 0x1f, 0x30, 0xab, 0xc0, 0xad, 0x97, 0x84, + 0x35, 0xa2, 0xa2, 0x94, 0x72, 0x1d, 0x69, 0x43, 0xf9, 0x2b, 0x7f, 0xe8, 0xf6, 0x6f, 0xac, 0x22, + 0xb7, 0x32, 0x85, 0xd5, 0x6f, 0x1c, 0x37, 0x14, 0x72, 0x2a, 0xf5, 0xe4, 0x3d, 0xa8, 0x3d, 0xf7, + 0x83, 0x6b, 0x27, 0x18, 0xf4, 0x7c, 0xab, 0xd4, 0x32, 0xda, 0x35, 0x1a, 0x0b, 0x48, 0x1b, 0x96, + 0x7b, 0xce, 0xe9, 0x90, 0x1d, 0xb0, 0xb7, 0x9d, 0x73, 0xc7, 0x3b, 0x63, 0x03, 0x6b, 0xb1, 0x65, + 0xb4, 0xab, 0x34, 0x29, 0x46, 0x1c, 0xca, 0xc2, 0xe0, 0x06, 0x5d, 0x58, 0xe5, 0x96, 0xd1, 0x2e, + 0xd2, 0x58, 0x40, 0xd6, 0x60, 0xf1, 0x30, 0xf0, 0x27, 0x23, 0xab, 0xd2, 0x32, 0xda, 0x4d, 0x2a, + 0x06, 0x64, 0x07, 0xaa, 0xdd, 0x73, 0x27, 0x18, 0xb8, 0xde, 0x99, 0x55, 0x55, 0x77, 0x13, 0x49, + 0xe9, 0x4c, 0x4f, 0x1e, 0x03, 0x74, 0x3d, 0x67, 0xd4, 0x3d, 0xf7, 0xc3, 0xde, 0xd8, 0xaa, 0xb5, + 0x8c, 0x76, 0x7d, 0x6f, 0xed, 0x7e, 0x1c, 0xe0, 0x5e, 0xf4, 0xf4, 0xb4, 0xf4, 0xcd, 0x3f, 0xee, + 0x2d, 0x50, 0xc5, 0x1a, 0x77, 0x81, 0xf1, 0xc1, 0x95, 0xa0, 0x99, 0x3f, 0x09, 0x2d, 0xe0, 0x2b, + 0x4c, 0x8a, 0xed, 0xff, 0x1a, 0x50, 0x43, 0x19, 0xdf, 0x1d, 0xae, 0x9a, 0x3f, 0xf0, 0x83, 0x29, + 0x51, 0x31, 0xc0, 0x9d, 0x76, 0x59, 0x70, 0xe5, 0xf6, 0xd9, 0xd1, 0x01, 0x3f, 0x84, 0x1a, 0x8d, + 0x05, 0xc4, 0x82, 0xca, 0x1b, 0x16, 0x8c, 0x5d, 0xdf, 0xe3, 0xa1, 0x2f, 0xd1, 0x68, 0x88, 0x68, + 0x6f, 0x9c, 0xa1, 0x3b, 0xe0, 0x51, 0xae, 0x52, 0x31, 0x88, 0x23, 0xb3, 0x98, 0x17, 0x99, 0xf2, + 0x9c, 0xc8, 0xb4, 0xa0, 0x7e, 0x12, 0xb8, 0x67, 0xae, 0x27, 0xd6, 0x5a, 0xe1, 0x5e, 0x55, 0x11, + 0x5a, 0xec, 0x0f, 0x87, 0x7e, 0xdf, 0x09, 0xfd, 0xe0, 0xe8, 0x80, 0x87, 0xba, 0x46, 0x55, 0x91, + 0xfd, 0x47, 0x80, 0x0a, 0x65, 0xbf, 0x9b, 0xb0, 0x71, 0x28, 0x4e, 0x92, 0x3f, 0x1e, 0x1d, 0xc8, + 0x9d, 0xc7, 0x02, 0xf2, 0x50, 0x09, 0x10, 0xdf, 0x7d, 0x7d, 0x6f, 0x39, 0xbe, 0x82, 0x5c, 0x2c, + 0x4f, 0x40, 0x09, 0xe4, 0x36, 0x94, 0x8f, 0x59, 0x78, 0xee, 0x0f, 0xe4, 0x75, 0x6c, 0x88, 0x19, + 0x42, 0x46, 0xa5, 0x8e, 0x7c, 0x04, 0x25, 0x9c, 0xc2, 0xe3, 0x53, 0x8f, 0x5e, 0x03, 0x94, 0x48, + 0xef, 0x12, 0x97, 0x1b, 0x91, 0x07, 0x50, 0x7e, 0xed, 0xa1, 0x05, 0x0f, 0x5c, 0x7d, 0x6f, 0x55, + 0x98, 0x0b, 0x99, 0x3e, 0x41, 0x1a, 0x92, 0x5f, 0x01, 0x1c, 0xb2, 0xb0, 0x37, 0xf5, 0xb8, 0x97, + 0x32, 0x9f, 0x76, 0x57, 0xbe, 0x6c, 0x33, 0xb9, 0x3e, 0x55, 0x99, 0x40, 0x8e, 0x60, 0xe9, 0x90, + 0x85, 0x78, 0x5b, 0x5c, 0xef, 0xec, 0xa5, 0x3b, 0x0e, 0x79, 0xa8, 0xeb, 0x7b, 0x3f, 0x9a, 0x41, + 0x28, 0x3a, 0x1d, 0x26, 0x31, 0x91, 0xfc, 0x1c, 0x2a, 0x87, 0x2c, 0x7c, 0xea, 0x7a, 0x03, 0x7e, + 0x18, 0x78, 0x93, 0x23, 0x0c, 0x14, 0xea, 0x93, 0x23, 0x53, 0x42, 0x61, 0xe5, 0x05, 0x63, 0xa3, + 0x38, 0xce, 0x38, 0x5f, 0xbc, 0x09, 0x5b, 0x62, 0x7e, 0x4a, 0xad, 0x23, 0xa5, 0xa7, 0xe3, 0xa6, + 0x50, 0x48, 0xd9, 0xa5, 0x1f, 0x32, 0x1e, 0x17, 0x50, 0x37, 0xa5, 0xeb, 0x12, 0x9b, 0xd2, 0x95, + 0xe4, 0x25, 0x2c, 0xf3, 0x2b, 0xed, 0x84, 0x4c, 0xbe, 0x0e, 0x56, 0x9d, 0x63, 0xbd, 0x27, 0xb0, + 0x12, 0x4a, 0x1d, 0x2c, 0x39, 0x95, 0x74, 0xa0, 0xd1, 0x71, 0x3c, 0xcf, 0x0f, 0x3b, 0xfe, 0xe5, + 0xa5, 0x1b, 0x5a, 0x0d, 0x0e, 0xb5, 0x21, 0xa0, 0x54, 0x8d, 0x8e, 0xa3, 0x4d, 0x42, 0x90, 0x43, + 0x16, 0xee, 0xf7, 0x43, 0xf7, 0x8a, 0xf5, 0xa6, 0x9e, 0xd5, 0x54, 0x41, 0x54, 0x4d, 0x02, 0x44, + 0x55, 0x61, 0xd8, 0xbb, 0x2c, 0xa4, 0xc8, 0x2e, 0x41, 0x18, 0xed, 0x6c, 0x49, 0x0d, 0x7b, 0x4a, + 0x9d, 0x08, 0x7b, 0x4a, 0x8f, 0x98, 0x1d, 0xc7, 0x4b, 0x60, 0x2e, 0xab, 0x98, 0x29, 0x75, 0x02, + 0x33, 0xa5, 0x27, 0xaf, 0x81, 0x50, 0x76, 0xe9, 0xb8, 0x5e, 0x6f, 0xea, 0x1d, 0x79, 0x11, 0xa8, + 0xc9, 0x41, 0xef, 0x09, 0xd0, 0xb4, 0x5e, 0x47, 0xcd, 0x00, 0x20, 0xbf, 0x86, 0x7a, 0xe7, 0x9c, + 0xf5, 0x2f, 0x4e, 0x82, 0xd1, 0xb9, 0xe3, 0x59, 0x2b, 0x1c, 0xcf, 0x92, 0x8b, 0x8c, 0x15, 0x3a, + 0x90, 0x3a, 0x05, 0x2f, 0x06, 0x65, 0xe3, 0xc9, 0x25, 0x3b, 0xf2, 0xae, 0xf0, 0x94, 0x3b, 0xaf, + 0x2c, 0xa2, 0x5e, 0x8c, 0x84, 0x32, 0x71, 0x31, 0x12, 0x5a, 0xe2, 0xc0, 0xfa, 0xfe, 0xa9, 0x1f, + 0x84, 0xe2, 0xe6, 0x1d, 0x30, 0x67, 0x80, 0x20, 0x78, 0xba, 0x6b, 0x1c, 0xf4, 0x27, 0x02, 0x34, + 0xdb, 0x46, 0xc7, 0xce, 0x01, 0xb2, 0xbf, 0x06, 0xa8, 0x52, 0x36, 0x1e, 0xf9, 0xde, 0x98, 0xcd, + 0xa1, 0xc3, 0x98, 0xd9, 0x0a, 0xb7, 0x30, 0xdb, 0x1a, 0x2c, 0x3e, 0x0b, 0x02, 0x3f, 0xe0, 0xf4, + 0xd7, 0xa0, 0x62, 0x40, 0x3e, 0x84, 0xca, 0x2b, 0x76, 0xcd, 0xdf, 0xe2, 0x52, 0x26, 0x91, 0xd2, + 0x48, 0x4f, 0x7e, 0x2a, 0xa9, 0x51, 0x70, 0x1d, 0x51, 0xa9, 0x51, 0x2c, 0x53, 0xe3, 0xc6, 0xbd, + 0x19, 0x37, 0x96, 0x55, 0x76, 0x89, 0xb8, 0x51, 0x9b, 0x11, 0x91, 0xe3, 0x13, 0x8d, 0x1c, 0x2b, + 0xea, 0x29, 0xab, 0xe4, 0xa8, 0xcd, 0x55, 0xd9, 0xf1, 0xcb, 0x14, 0x3b, 0x56, 0xd5, 0x33, 0x4e, + 0xb2, 0xa3, 0x86, 0x93, 0xa4, 0xc7, 0x47, 0x31, 0x3d, 0x0a, 0x7a, 0xbb, 0x93, 0xa0, 0x47, 0x6d, + 0xf6, 0x8c, 0x1f, 0xbb, 0x59, 0xfc, 0x08, 0xea, 0xfd, 0xcf, 0xe0, 0x47, 0x0d, 0x2a, 0x83, 0x20, + 0xbf, 0x4c, 0x11, 0xa4, 0x46, 0x6a, 0x49, 0x82, 0xd4, 0xf7, 0x95, 0x60, 0xc8, 0xe3, 0x34, 0x43, + 0x0a, 0x5a, 0x7b, 0x3f, 0x87, 0x21, 0x35, 0xb4, 0x14, 0x45, 0x1e, 0x24, 0x28, 0x52, 0xb0, 0xdb, + 0x66, 0x16, 0x45, 0x6a, 0x40, 0x3a, 0x47, 0x1e, 0x24, 0x38, 0x72, 0x49, 0x45, 0xd1, 0x39, 0x52, + 0x47, 0xd1, 0x48, 0xb2, 0x9b, 0x45, 0x92, 0xcb, 0x6a, 0xec, 0x33, 0x48, 0x52, 0x8f, 0x7d, 0x9a, + 0x25, 0xbb, 0x59, 0x2c, 0xa9, 0x11, 0x5a, 0x06, 0x4b, 0xea, 0xa0, 0x69, 0x9a, 0x7c, 0x93, 0x49, + 0x93, 0x82, 0xd6, 0x5a, 0xf9, 0x34, 0xa9, 0xc1, 0x66, 0xf1, 0xe4, 0xbe, 0xce, 0x93, 0x44, 0xcb, + 0x57, 0x2a, 0x4f, 0x6a, 0x48, 0x1a, 0x51, 0x1e, 0xa7, 0x89, 0x72, 0x55, 0xbd, 0x1f, 0x29, 0xa2, + 0xd4, 0xef, 0x47, 0x92, 0x29, 0x4f, 0xe7, 0x30, 0xe5, 0xf6, 0xed, 0x4c, 0xa9, 0x81, 0xe7, 0x51, + 0xe5, 0x1f, 0x0c, 0xd1, 0xd1, 0x44, 0xc5, 0x23, 0x96, 0xcc, 0x53, 0x4f, 0x32, 0x65, 0x83, 0x8a, + 0xc1, 0x9c, 0x92, 0x99, 0x40, 0x89, 0xfa, 0xd7, 0x63, 0xab, 0xd8, 0x2a, 0xb6, 0x1b, 0x94, 0x3f, + 0x93, 0x07, 0x50, 0x91, 0x4d, 0x52, 0xba, 0x1c, 0x94, 0x8a, 0xe8, 0xf5, 0x97, 0x43, 0xfb, 0x31, + 0x34, 0xd4, 0x77, 0x90, 0xec, 0x40, 0x19, 0x23, 0x32, 0x0c, 0xf9, 0x5a, 0xea, 0x11, 0x35, 0x0b, + 0x59, 0xc4, 0x7e, 0x62, 0x64, 0x7f, 0x0e, 0x2b, 0xa9, 0x12, 0x30, 0x67, 0x2f, 0x26, 0x14, 0xa9, + 0x7f, 0xcd, 0x77, 0xd1, 0xa0, 0xf8, 0x68, 0x3b, 0x40, 0xd2, 0x14, 0x29, 0xcb, 0xfd, 0x89, 0x68, + 0x1e, 0x16, 0xa9, 0x18, 0x90, 0x47, 0x50, 0x57, 0x39, 0xb2, 0xd0, 0x2a, 0xb6, 0xeb, 0x7b, 0xcd, + 0xb8, 0x3b, 0xeb, 0x4d, 0xbd, 0xe8, 0x66, 0x28, 0x76, 0xf6, 0x13, 0xb8, 0x93, 0x59, 0x5f, 0x92, + 0x0f, 0xa0, 0x88, 0x07, 0x2a, 0x76, 0x98, 0x89, 0x83, 0x7a, 0xfb, 0x04, 0xd6, 0xb3, 0x19, 0x38, + 0xb9, 0x20, 0xe3, 0x3b, 0x2e, 0xa8, 0x0f, 0x15, 0xa9, 0xcd, 0x3f, 0xf2, 0x4e, 0xc0, 0x9c, 0x90, + 0x0d, 0x4e, 0xbc, 0xe8, 0xc8, 0x67, 0x02, 0xb2, 0x0d, 0x4d, 0x9c, 0xce, 0x82, 0xfd, 0xc1, 0x20, + 0x60, 0xe3, 0x31, 0x4f, 0x8c, 0x35, 0xaa, 0x0b, 0xed, 0x3f, 0x1b, 0xd0, 0xd4, 0x0a, 0xfa, 0x1c, + 0x5f, 0x9f, 0x40, 0x55, 0x90, 0x59, 0xaf, 0x2b, 0x5b, 0x92, 0xdb, 0x3a, 0xc3, 0x99, 0x2d, 0xf9, + 0x14, 0x6a, 0xc7, 0x93, 0xd0, 0x11, 0xd7, 0xac, 0xc8, 0x77, 0x2e, 0xdb, 0x88, 0x67, 0xd3, 0x30, + 0x70, 0x22, 0x5d, 0xd4, 0xcf, 0xcc, 0x6c, 0x6d, 0x13, 0x96, 0xf4, 0x64, 0x6a, 0x7f, 0x6d, 0xf0, + 0xfc, 0xa7, 0xd4, 0xdc, 0xfa, 0xa5, 0x37, 0x92, 0x97, 0x7e, 0xd6, 0x5b, 0x16, 0xd4, 0xde, 0x72, + 0xd6, 0x0d, 0x16, 0xf3, 0xba, 0xc1, 0xd2, 0xf7, 0xeb, 0x06, 0x17, 0x53, 0xdd, 0xa0, 0xfd, 0x27, + 0x03, 0x96, 0x13, 0x99, 0xf4, 0xff, 0xeb, 0xea, 0x76, 0xc0, 0x9c, 0xf5, 0x90, 0x7a, 0xcf, 0x9b, + 0x92, 0x27, 0x5b, 0xd0, 0x52, 0xba, 0x05, 0xfd, 0xbb, 0x01, 0x56, 0x5e, 0xff, 0x32, 0x27, 0x96, + 0xdb, 0x50, 0xee, 0x86, 0x4e, 0x38, 0x19, 0xeb, 0x45, 0x98, 0x90, 0x51, 0xa9, 0x23, 0xeb, 0x50, + 0xe6, 0xd7, 0x25, 0x22, 0x1a, 0x39, 0x22, 0x8f, 0x00, 0x66, 0x3e, 0x91, 0x6d, 0x8a, 0xf9, 0x9b, + 0x57, 0x0c, 0xed, 0xbf, 0x1a, 0xb0, 0x91, 0x5b, 0x4f, 0x90, 0x25, 0x28, 0x9c, 0xbc, 0xe0, 0x2b, + 0xad, 0xd2, 0xc2, 0xc9, 0x8b, 0xef, 0xb8, 0xc4, 0x1f, 0x36, 0xa2, 0x9f, 0x81, 0x95, 0xd7, 0x99, + 0xdc, 0x1e, 0x50, 0xfb, 0x23, 0xd8, 0xc8, 0x4d, 0xd7, 0xc9, 0xad, 0xa1, 0x9b, 0xbc, 0x66, 0x65, + 0xbe, 0x9b, 0xdc, 0x04, 0x9e, 0x72, 0xf3, 0x0b, 0xd8, 0xc8, 0x6d, 0x5f, 0xe6, 0xf8, 0x79, 0x0c, + 0x9b, 0xf9, 0x29, 0x5d, 0x14, 0xf8, 0x52, 0x2b, 0xc9, 0x3a, 0x16, 0xd8, 0x8f, 0xe0, 0x4e, 0x66, + 0x13, 0x3c, 0xc7, 0x65, 0x1b, 0xd6, 0xb3, 0x4b, 0xc3, 0xd4, 0xbe, 0x3e, 0x81, 0xf5, 0xec, 0xce, + 0x78, 0x8e, 0x87, 0x0f, 0xe1, 0x6e, 0x4e, 0xbd, 0x98, 0x72, 0xf1, 0x1c, 0xde, 0xbf, 0xb5, 0x1d, + 0xc2, 0x2c, 0x12, 0xce, 0xc9, 0x22, 0xe1, 0xd4, 0xb3, 0x7f, 0x06, 0x5b, 0xb7, 0x17, 0x0b, 0x29, + 0xcf, 0x14, 0x56, 0x33, 0x7a, 0x75, 0xf2, 0x39, 0x34, 0x45, 0xc9, 0x83, 0x49, 0x33, 0x4e, 0x3b, + 0xf2, 0xad, 0x9b, 0xa9, 0xa4, 0x6f, 0xdd, 0xd6, 0xfe, 0x25, 0xac, 0x65, 0x15, 0xb7, 0x98, 0x53, + 0x84, 0x04, 0x93, 0x94, 0x38, 0x4b, 0x7c, 0xcd, 0x75, 0xa1, 0xfd, 0x10, 0x56, 0x33, 0x1a, 0xff, + 0x39, 0xb1, 0x7e, 0x02, 0x6b, 0x59, 0x95, 0x70, 0xfc, 0x49, 0xcf, 0x50, 0x3f, 0xe9, 0x99, 0x22, + 0x27, 0x17, 0xb8, 0x7b, 0x9e, 0x7e, 0x0f, 0x80, 0xa4, 0x5b, 0xe5, 0x39, 0xa4, 0x36, 0x43, 0x31, + 0x22, 0x94, 0x8f, 0x61, 0x35, 0xa3, 0x90, 0x44, 0x5e, 0x93, 0x35, 0xa7, 0x58, 0x85, 0x1c, 0xd9, + 0x9f, 0x42, 0x6d, 0x16, 0x38, 0x62, 0x41, 0x25, 0x2a, 0x75, 0x85, 0xa7, 0x68, 0x98, 0xb1, 0xda, + 0xdf, 0x17, 0xa3, 0xca, 0x89, 0x3c, 0x80, 0x2a, 0x5e, 0x5e, 0x9e, 0xc4, 0x8d, 0xdb, 0xd2, 0xc2, + 0xcc, 0x0c, 0x79, 0xe9, 0x0b, 0x67, 0xdc, 0xf1, 0xbd, 0xb7, 0x43, 0xb7, 0x1f, 0xf2, 0xf5, 0x57, + 0xa9, 0x2a, 0xc2, 0x83, 0xfa, 0xc2, 0x19, 0x7f, 0x15, 0xb0, 0x2b, 0xd9, 0xb8, 0x14, 0xb9, 0x8d, + 0x2e, 0x24, 0x9f, 0x41, 0x6d, 0x96, 0xb9, 0x65, 0x0d, 0x78, 0x5b, 0x56, 0x8f, 0x8d, 0xbf, 0xc7, + 0x47, 0xeb, 0x16, 0xd4, 0xa3, 0x55, 0xbd, 0x60, 0x37, 0xbc, 0x5b, 0x6e, 0x50, 0x55, 0xa4, 0x5a, + 0x60, 0x94, 0x2a, 0xba, 0x05, 0x46, 0x76, 0x0b, 0x00, 0x57, 0x2d, 0x2a, 0x17, 0xde, 0xf4, 0x36, + 0xa8, 0x22, 0xc1, 0xc8, 0x8b, 0x27, 0xf1, 0xd5, 0xba, 0x49, 0xa3, 0x21, 0xce, 0x7c, 0xc5, 0xae, + 0x31, 0x70, 0xfb, 0x03, 0xd1, 0xa8, 0x56, 0xa9, 0x22, 0xb1, 0xbb, 0xd0, 0xd4, 0xea, 0x10, 0x3c, + 0xaa, 0x0b, 0x76, 0x23, 0x6b, 0x1f, 0x7c, 0xc4, 0xd2, 0x79, 0x7c, 0xe1, 0x8e, 0x64, 0x94, 0xf9, + 0x33, 0x5e, 0xab, 0x80, 0x8d, 0x86, 0x4e, 0x9f, 0xf5, 0x7c, 0xf9, 0xc1, 0x21, 0x16, 0x20, 0xdd, + 0x64, 0x7f, 0x6f, 0x99, 0xf3, 0x0a, 0x6c, 0xc0, 0xdd, 0x9c, 0xf6, 0x63, 0xe7, 0xc7, 0xda, 0x5f, + 0x0c, 0x52, 0xe1, 0x05, 0xb2, 0xb9, 0x40, 0x6a, 0xb0, 0x48, 0x31, 0xce, 0xa6, 0xb1, 0xf3, 0x81, + 0xb8, 0x47, 0xfc, 0xdf, 0x44, 0x13, 0x6a, 0xcf, 0xa6, 0xfd, 0xe1, 0x64, 0xec, 0x5e, 0x31, 0x73, + 0x81, 0x00, 0x94, 0xb1, 0x78, 0x61, 0x03, 0xd3, 0xd8, 0xd9, 0x06, 0x88, 0x7f, 0x51, 0x90, 0x2a, + 0x94, 0x70, 0x64, 0x2e, 0x90, 0x06, 0x54, 0x9f, 0x3b, 0xe3, 0xf0, 0xb9, 0xe3, 0x0e, 0x4d, 0x63, + 0xe7, 0x5e, 0x5c, 0x0e, 0xa1, 0xcd, 0x2b, 0xdf, 0x63, 0xc2, 0xdb, 0xd3, 0x1b, 0x74, 0x6c, 0xec, + 0xfc, 0xb3, 0x10, 0x7d, 0x95, 0x41, 0x3d, 0x3a, 0x16, 0x7e, 0x44, 0xcd, 0x66, 0x1a, 0x64, 0x49, + 0xfd, 0xd8, 0x61, 0x16, 0x08, 0x49, 0x7e, 0xbc, 0x30, 0x8b, 0x28, 0xd3, 0x19, 0xdc, 0x2c, 0x91, + 0xfa, 0xec, 0xc3, 0x84, 0xb9, 0x48, 0xee, 0x64, 0x7c, 0x6e, 0x30, 0xcb, 0x64, 0x19, 0xea, 0xf2, + 0xff, 0x09, 0x9f, 0x54, 0x21, 0x2b, 0xd0, 0x94, 0x02, 0xe9, 0xbf, 0x4a, 0x56, 0x53, 0x1f, 0x02, + 0xcc, 0x1a, 0x31, 0xf5, 0x76, 0xde, 0x04, 0x94, 0xa8, 0xb4, 0x63, 0xd6, 0xd1, 0x67, 0x2a, 0x31, + 0x9b, 0x0d, 0xb2, 0x9e, 0xd5, 0xd3, 0x9a, 0x4d, 0x34, 0x4f, 0x25, 0x58, 0x73, 0x09, 0x97, 0xa8, + 0x10, 0x89, 0xb9, 0x8c, 0xeb, 0x49, 0x1c, 0xae, 0x69, 0x92, 0xcd, 0xbc, 0xf6, 0xd1, 0x5c, 0xd9, + 0x61, 0x51, 0x39, 0x23, 0x56, 0xc4, 0x81, 0x71, 0xbb, 0xcf, 0x3c, 0x8c, 0x84, 0xb9, 0x80, 0x2b, + 0x52, 0xc4, 0x32, 0xb2, 0xa6, 0xa1, 0x98, 0xbf, 0xe6, 0xc1, 0xef, 0x4e, 0xfa, 0x7d, 0xb3, 0xa0, + 0x88, 0xe3, 0xf5, 0x9a, 0xc5, 0xa7, 0x9d, 0x6f, 0xff, 0xbd, 0x65, 0x7c, 0xf3, 0x6e, 0xcb, 0xf8, + 0xf6, 0xdd, 0x96, 0xf1, 0xaf, 0x77, 0x5b, 0x0b, 0x7f, 0xfb, 0xcf, 0x96, 0xf1, 0x5b, 0xf5, 0x2f, + 0xdc, 0xa5, 0x13, 0x06, 0xee, 0xd4, 0xe7, 0xe5, 0x6d, 0x34, 0xf0, 0xd8, 0xee, 0xe8, 0xe2, 0x6c, + 0x77, 0x74, 0xba, 0x8b, 0x0b, 0x3e, 0x2d, 0xf3, 0x7f, 0x6f, 0x0f, 0xff, 0x17, 0x00, 0x00, 0xff, + 0xff, 0xb1, 0xc7, 0x79, 0x20, 0xcf, 0x1b, 0x00, 0x00, } func (m *LockOptions) Marshal() (dAtA []byte, err error) { @@ -3155,6 +3181,13 @@ func (m *LockTable) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.AllocatorID) > 0 { + i -= len(m.AllocatorID) + copy(dAtA[i:], m.AllocatorID) + i = encodeVarintLock(dAtA, i, uint64(len(m.AllocatorID))) + i-- + dAtA[i] = 0x42 + } if m.OriginTable != 0 { i = encodeVarintLock(dAtA, i, uint64(m.OriginTable)) i-- @@ -4114,6 +4147,13 @@ func (m *GetBindResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.AllocatorID) > 0 { + i -= len(m.AllocatorID) + copy(dAtA[i:], m.AllocatorID) + i = encodeVarintLock(dAtA, i, uint64(len(m.AllocatorID))) + i-- + dAtA[i] = 0x22 + } if m.AllocatorVersion != 0 { i = encodeVarintLock(dAtA, i, uint64(m.AllocatorVersion)) i-- @@ -4218,6 +4258,13 @@ func (m *KeepLockTableBindResponse) MarshalToSizedBuffer(dAtA []byte) (int, erro i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.AllocatorID) > 0 { + i -= len(m.AllocatorID) + copy(dAtA[i:], m.AllocatorID) + i = encodeVarintLock(dAtA, i, uint64(len(m.AllocatorID))) + i-- + dAtA[i] = 0x22 + } if m.AllocatorVersion != 0 { i = encodeVarintLock(dAtA, i, uint64(m.AllocatorVersion)) i-- @@ -5249,6 +5296,10 @@ func (m *LockTable) ProtoSize() (n int) { if m.OriginTable != 0 { n += 1 + sovLock(uint64(m.OriginTable)) } + l = len(m.AllocatorID) + if l > 0 { + n += 1 + l + sovLock(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5579,6 +5630,10 @@ func (m *GetBindResponse) ProtoSize() (n int) { if m.AllocatorVersion != 0 { n += 1 + sovLock(uint64(m.AllocatorVersion)) } + l = len(m.AllocatorID) + if l > 0 { + n += 1 + l + sovLock(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5631,6 +5686,10 @@ func (m *KeepLockTableBindResponse) ProtoSize() (n int) { if m.AllocatorVersion != 0 { n += 1 + sovLock(uint64(m.AllocatorVersion)) } + l = len(m.AllocatorID) + if l > 0 { + n += 1 + l + sovLock(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6496,6 +6555,38 @@ func (m *LockTable) Unmarshal(dAtA []byte) error { break } } + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AllocatorID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLock + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLock + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLock + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AllocatorID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLock(dAtA[iNdEx:]) @@ -9105,6 +9196,38 @@ func (m *GetBindResponse) Unmarshal(dAtA []byte) error { break } } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AllocatorID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLock + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLock + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLock + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AllocatorID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLock(dAtA[iNdEx:]) @@ -9382,6 +9505,38 @@ func (m *KeepLockTableBindResponse) Unmarshal(dAtA []byte) error { break } } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AllocatorID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLock + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLock + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLock + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AllocatorID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLock(dAtA[iNdEx:]) diff --git a/proto/lock.proto b/proto/lock.proto index dd2a88ffd2ea9..62db84aa2fbe4 100644 --- a/proto/lock.proto +++ b/proto/lock.proto @@ -76,6 +76,8 @@ message LockTable { uint32 Group = 5; Sharding Sharding = 6; uint64 OriginTable = 7; + // AllocatorID identifies the allocator process that created this bind. + string AllocatorID = 8; } // Method lock table operations @@ -253,6 +255,7 @@ message GetBindRequest { message GetBindResponse { LockTable LockTable = 2 [(gogoproto.nullable) = false]; uint64 AllocatorVersion = 3; + string AllocatorID = 4; } // KeepLockTableBindRequest keep lock table bind request. CN -> TN @@ -268,6 +271,7 @@ message KeepLockTableBindResponse { bool OK = 1; Status Status = 2; uint64 AllocatorVersion = 3; + string AllocatorID = 4; } // SetRestartServiceRequest set restart lock service request from allocator request. operator -> TN From 870d2195e9923df2c1e1adf5250d0e9f80408283 Mon Sep 17 00:00:00 2001 From: iamlinjunhong <1030420200@qq.com> Date: Wed, 10 Jun 2026 16:00:36 +0800 Subject: [PATCH 4/6] fix(lockservice): remove unused allocator observer helper --- pkg/lockservice/service.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/lockservice/service.go b/pkg/lockservice/service.go index f34c0e62a4fae..7c401df36095b 100644 --- a/pkg/lockservice/service.go +++ b/pkg/lockservice/service.go @@ -676,14 +676,6 @@ func (s *service) observeAllocatorState(source string, observed allocatorState) return s.observeAllocatorStateWithHolders(source, observed, s.tableGroups) } -func (s *service) observeAllocatorVersionWithHolders( - source string, - observedVersion uint64, - holders *lockTableHolders, -) int { - return s.observeAllocatorStateWithHolders(source, allocatorState{version: observedVersion}, holders) -} - func (s *service) observeAllocatorStateWithHolders( source string, observed allocatorState, From 6e8ca4b01b0b827dfb1a01e7c5b4cefffa38b869 Mon Sep 17 00:00:00 2001 From: iamlinjunhong <1030420200@qq.com> Date: Wed, 10 Jun 2026 17:16:59 +0800 Subject: [PATCH 5/6] fix(lockservice): keep remote lock bookkeeping on timeout --- pkg/lockservice/lock_table_remote.go | 34 ++---------- pkg/lockservice/lock_table_remote_test.go | 68 +++++++++++++++++++++++ 2 files changed, 72 insertions(+), 30 deletions(-) diff --git a/pkg/lockservice/lock_table_remote.go b/pkg/lockservice/lock_table_remote.go index e0d04ef892937..e8c416b99aa7e 100644 --- a/pkg/lockservice/lock_table_remote.go +++ b/pkg/lockservice/lock_table_remote.go @@ -165,12 +165,10 @@ func (l *remoteLockTable) lock( return } - // Only record the lock on errors when the request could have been observed - // by the remote side. For local context timeouts / cancellations, the - // request may never be sent, and recording a phantom lock will leak waiters. - if !skipTrackLockOnError(ctx, err) { - _ = txn.lockAdded(l.bind.Group, l.bind, rows, l.logger) - } + // The request may have reached the remote owner and acquired locks even if + // the response was lost or the client-side context timed out. Keep local + // bookkeeping so normal transaction close can send the remote unlock. + _ = txn.lockAdded(l.bind.Group, l.bind, rows, l.logger) logRemoteLockFailed(l.logger, txn, rows, opts, l.bind, err) // encounter any error, we need try to check bind is valid. // And use origin error to return, because once handlerError @@ -388,30 +386,6 @@ func (l *remoteLockTable) handleError( return oldError } -// skipTrackLockOnError returns true when the error indicates the request -// definitely did not reach the remote lock service (e.g. local context -// cancellation/timeout). In such cases, recording the lock would create a -// phantom holder that can never be cleaned up. -// -// If the request actually reached the remote side before timeout, the remote -// lock will be cleaned up by the keepRemoteLock timeout mechanism (bounded by -// RemoteLockTimeout, default 10m), so skipping the local bookkeeping is safe -// though it may introduce a bounded extra wait. -func skipTrackLockOnError(ctx context.Context, err error) bool { - if ctxErr := ctx.Err(); ctxErr != nil { - if errors.Is(ctxErr, context.DeadlineExceeded) || - errors.Is(ctxErr, context.Canceled) { - return true - } - } - if errors.Is(err, context.DeadlineExceeded) || - errors.Is(err, context.Canceled) || - moerr.IsMoErrCode(err, moerr.ErrRPCTimeout) { - return true - } - return false -} - func retryRemoteLockError(err error) bool { var netErr net.Error if errors.As(err, &netErr) && netErr.Timeout() { diff --git a/pkg/lockservice/lock_table_remote_test.go b/pkg/lockservice/lock_table_remote_test.go index 6653bfaa16d3d..559f1a9865a58 100644 --- a/pkg/lockservice/lock_table_remote_test.go +++ b/pkg/lockservice/lock_table_remote_test.go @@ -99,6 +99,74 @@ func TestIssue20747(t *testing.T) { ) } +func TestLockRemoteWithContextTimeoutTracksLockForUnlock(t *testing.T) { + unlockCalled := make(chan struct{}) + runRemoteLockTableTests( + t, + pb.LockTable{ServiceID: "s1"}, + func(s Server) { + s.RegisterMethodHandler( + pb.Method_Lock, + func( + ctx context.Context, + cancel context.CancelFunc, + req *pb.Request, + resp *pb.Response, + cs morpc.ClientSession) { + // Simulate a slow or dropped response. The server just waits for + // the client context to expire. + <-ctx.Done() + }, + ) + s.RegisterMethodHandler( + pb.Method_Unlock, + func( + ctx context.Context, + cancel context.CancelFunc, + req *pb.Request, + resp *pb.Response, + cs morpc.ClientSession) { + close(unlockCalled) + writeResponse(getLogger(""), cancel, resp, nil, cs) + }, + ) + }, + func(l *remoteLockTable, s Server) { + txnID := []byte("txn-timeout") + txn := newActiveTxn(txnID, string(txnID), newFixedSlicePool(32), "") + closed := false + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + txn.Lock() + defer func() { + txn.Unlock() + if !closed { + reuse.Free(txn, nil) + } + }() + + l.lock(ctx, txn, [][]byte{{1}}, LockOptions{}, func(r pb.Result, err error) { + require.Error(t, err) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrBackendCannotConnect)) + }) + holder := txn.getHoldLocksLocked(l.bind.Group) + require.Contains(t, holder.tableKeys, l.bind.Table) + require.Contains(t, holder.tableBinds, l.bind.Table) + + require.NoError(t, txn.close(txnID, timestamp.Timestamp{}, func(uint32, uint64) (lockTable, error) { + return l, nil + }, l.logger)) + closed = true + select { + case <-unlockCalled: + default: + require.Fail(t, "expected remote unlock") + } + }, + func(lt pb.LockTable) {}, + ) +} + func TestLockRemoteWithNeedUpgrade(t *testing.T) { runRemoteLockTableTests( t, From 30a092f7a1975d148e0acc4fa88c103a00a8cb08 Mon Sep 17 00:00:00 2001 From: iamlinjunhong <1030420200@qq.com> Date: Thu, 11 Jun 2026 10:51:54 +0800 Subject: [PATCH 6/6] fix(lockservice): fence keepalive purge and reject stale allocator --- pkg/lockservice/lock_table_keeper.go | 12 ++- pkg/lockservice/service.go | 121 +++++++++++++++++++++++---- pkg/lockservice/service_test.go | 114 +++++++++++++++++++++++++ pkg/pb/lock/lock.go | 8 +- 4 files changed, 233 insertions(+), 22 deletions(-) diff --git a/pkg/lockservice/lock_table_keeper.go b/pkg/lockservice/lock_table_keeper.go index d035bd60a20b7..f25b645458f91 100644 --- a/pkg/lockservice/lock_table_keeper.go +++ b/pkg/lockservice/lock_table_keeper.go @@ -185,6 +185,7 @@ func (k *lockTableKeeper) doKeepLockTableBind(ctx context.Context) { req.KeepLockTableBind.TxnIDs = k.service.activeTxnHolder.getAllTxnID() } + requestAllocator := k.service.allocatorStateSnapshot() ctx, cancel := context.WithTimeoutCause(ctx, k.keepLockTableBindInterval, moerr.CauseDoKeepLockTableBind) defer cancel() resp, err := k.client.Send(ctx, req) @@ -196,13 +197,17 @@ func (k *lockTableKeeper) doKeepLockTableBind(ctx context.Context) { defer releaseResponse(resp) if resp.KeepLockTableBind.OK { - k.service.observeAllocatorStateWithHolders( + if _, accepted := k.service.observeAllocatorStateWithHoldersFromSnapshot( "keepalive", allocatorState{ id: resp.KeepLockTableBind.AllocatorID, version: resp.KeepLockTableBind.AllocatorVersion, }, - k.groupTables) + requestAllocator, + true, + k.groupTables); !accepted { + return + } switch resp.KeepLockTableBind.Status { case pb.Status_ServiceLockEnable: if !k.service.isStatus(pb.Status_ServiceLockEnable) { @@ -233,7 +238,8 @@ func (k *lockTableKeeper) doKeepLockTableBind(ctx context.Context) { allocatorState{ id: resp.KeepLockTableBind.AllocatorID, version: resp.KeepLockTableBind.AllocatorVersion, - }) + }, + requestAllocator) if n > 0 { // Keep bind receiving an explicit failure means that all the binds of the local diff --git a/pkg/lockservice/service.go b/pkg/lockservice/service.go index 7c401df36095b..039c8c7226e98 100644 --- a/pkg/lockservice/service.go +++ b/pkg/lockservice/service.go @@ -61,9 +61,10 @@ type service struct { fetchWhoWaitingListC chan who logger *log.MOLogger - allocatorVersionMu sync.Mutex - lastAllocatorVersion uint64 - lastAllocatorID string + allocatorVersionMu sync.Mutex + lastAllocatorVersion uint64 + lastAllocatorID string + supersededAllocatorIDs map[string]struct{} remote struct { client Client @@ -636,6 +637,7 @@ func (s *service) getLockTableWithCreate( close(c) }() + requestAllocator := s.allocatorStateSnapshot() bind, allocator, err := getLockTableBind( s.remote.client, group, @@ -647,7 +649,9 @@ func (s *service) getLockTableWithCreate( return nil, err } - s.observeAllocatorState("get-bind", allocator) + if _, accepted := s.observeAllocatorState("get-bind", allocator, requestAllocator); !accepted { + return nil, ErrLockTableBindChanged + } v := s.tableGroups.set(group, tableID, s.createLockTableByBind(bind)) return v, nil } @@ -672,8 +676,12 @@ func (s *service) observeAllocatorVersion(source string, observedVersion uint64) return s.observeAllocatorStateWithHolders(source, allocatorState{version: observedVersion}, s.tableGroups) } -func (s *service) observeAllocatorState(source string, observed allocatorState) int { - return s.observeAllocatorStateWithHolders(source, observed, s.tableGroups) +func (s *service) observeAllocatorState( + source string, + observed allocatorState, + requestAllocator allocatorState, +) (int, bool) { + return s.observeAllocatorStateWithHoldersFromSnapshot(source, observed, requestAllocator, true, s.tableGroups) } func (s *service) observeAllocatorStateWithHolders( @@ -681,18 +689,31 @@ func (s *service) observeAllocatorStateWithHolders( observed allocatorState, holders *lockTableHolders, ) int { + removed, _ := s.observeAllocatorStateWithHoldersFromSnapshot(source, observed, allocatorState{}, false, holders) + return removed +} + +func (s *service) observeAllocatorStateWithHoldersFromSnapshot( + source string, + observed allocatorState, + requestAllocator allocatorState, + hasRequestAllocator bool, + holders *lockTableHolders, +) (int, bool) { s.allocatorVersionMu.Lock() defer s.allocatorVersionMu.Unlock() - return s.observeAllocatorStateLocked(source, observed, holders) + return s.observeAllocatorStateLocked(source, observed, requestAllocator, hasRequestAllocator, holders) } func (s *service) observeAllocatorStateLocked( source string, observed allocatorState, + requestAllocator allocatorState, + hasRequestAllocator bool, holders *lockTableHolders, -) int { +) (int, bool) { if observed.version == 0 && observed.id == "" { - return 0 + return 0, true } oldVersion := s.lastAllocatorVersion @@ -700,13 +721,16 @@ func (s *service) observeAllocatorStateLocked( allocatorChanged := observed.id != "" && observed.id != oldID && (oldID != "" || oldVersion != 0) + if s.isAllocatorStateRejectedLocked(source, observed, requestAllocator, hasRequestAllocator, allocatorChanged, oldID, oldVersion) { + return 0, false + } if !allocatorChanged && oldVersion != 0 && observed.version < oldVersion { v2.GetLockServiceAllocatorEpochRegressionCounter(source).Inc() logAllocatorEpochRegression(s.logger, source, oldVersion, observed.version) - return 0 + return 0, false } if !allocatorChanged && observed.version == oldVersion { - return 0 + return 0, true } // Defensive path for minimal service instances used by keeper tests. @@ -716,7 +740,7 @@ func (s *service) observeAllocatorStateLocked( v2.LockServiceAllocatorEpochObservedGauge.Set(float64(observed.version)) v2.GetLockServiceAllocatorEpochChangedCounter(source).Inc() logAllocatorEpochChanged(s.logger, source, oldVersion, observed.version, 0) - return 0 + return 0, true } s.bindChangeMu.Lock() @@ -738,11 +762,17 @@ func (s *service) observeAllocatorStateLocked( v2.GetLockServiceStaleBindPurgedCounter(source).Add(float64(removed)) } logAllocatorEpochChanged(s.logger, source, oldVersion, observed.version, removed) - return removed + return removed, true } func (s *service) updateAllocatorStateLocked(observed allocatorState) { if observed.id != "" { + if s.lastAllocatorID != "" && s.lastAllocatorID != observed.id { + if s.supersededAllocatorIDs == nil { + s.supersededAllocatorIDs = make(map[string]struct{}) + } + s.supersededAllocatorIDs[s.lastAllocatorID] = struct{}{} + } s.lastAllocatorID = observed.id } if observed.version != 0 { @@ -750,6 +780,47 @@ func (s *service) updateAllocatorStateLocked(observed allocatorState) { } } +func (s *service) allocatorStateSnapshot() allocatorState { + s.allocatorVersionMu.Lock() + defer s.allocatorVersionMu.Unlock() + return allocatorState{ + id: s.lastAllocatorID, + version: s.lastAllocatorVersion, + } +} + +func (s *service) isAllocatorStateRejectedLocked( + source string, + observed allocatorState, + requestAllocator allocatorState, + hasRequestAllocator bool, + allocatorChanged bool, + oldID string, + oldVersion uint64, +) bool { + if observed.id != "" { + if _, ok := s.supersededAllocatorIDs[observed.id]; ok { + v2.GetLockServiceAllocatorEpochRegressionCounter(source).Inc() + logAllocatorEpochRegression(s.logger, source, oldVersion, observed.version) + return true + } + if hasRequestAllocator && + (requestAllocator.id != oldID || requestAllocator.version != oldVersion) && + oldID != "" && + observed.id != oldID { + v2.GetLockServiceAllocatorEpochRegressionCounter(source).Inc() + logAllocatorEpochRegression(s.logger, source, oldVersion, observed.version) + return true + } + } + if !allocatorChanged && oldVersion != 0 && observed.version < oldVersion { + v2.GetLockServiceAllocatorEpochRegressionCounter(source).Inc() + logAllocatorEpochRegression(s.logger, source, oldVersion, observed.version) + return true + } + return false +} + func (s *service) removeLockTablesWithFence( holders *lockTableHolders, filter func(pb.LockTable) bool, @@ -788,29 +859,41 @@ func (s *service) handleKeepBindFailed( holders *lockTableHolders, oldTableVersion uint64, allocator allocatorState, + requestAllocator allocatorState, ) int { s.allocatorVersionMu.Lock() defer s.allocatorVersionMu.Unlock() + if s.isAllocatorStateRejectedLocked( + "keepalive-ok-false-epoch", + allocator, + requestAllocator, + true, + allocator.id != "" && allocator.id != s.lastAllocatorID && (s.lastAllocatorID != "" || s.lastAllocatorVersion != 0), + s.lastAllocatorID, + s.lastAllocatorVersion) { + return 0 + } + if holders == nil { - s.observeAllocatorStateLocked("keepalive-ok-false-epoch", allocator, nil) + s.observeAllocatorStateLocked("keepalive-ok-false-epoch", allocator, requestAllocator, true, nil) return 0 } // Keep the original OK=false snapshot guard: if another local purge already // changed holders.version, this service-level purge should skip this round. s.bindChangeMu.Lock() - removed := holders.removeWithFilter(func(_ uint64, lt lockTable) bool { + removed := s.removeLockTablesWithFence(holders, func(bind pb.LockTable) bool { if oldTableVersion != holders.getVersion() { return false } - return lt.getBind().ServiceID == serviceID - }, closeReasonKeepBindFailed) + return bind.ServiceID == serviceID + }, allocator) s.bindChangeMu.Unlock() if removed > 0 { v2.GetLockServiceStaleBindPurgedCounter("keepalive-ok-false-service").Add(float64(removed)) } - s.observeAllocatorStateLocked("keepalive-ok-false-epoch", allocator, holders) + s.observeAllocatorStateLocked("keepalive-ok-false-epoch", allocator, requestAllocator, true, holders) return removed } @@ -1233,6 +1316,8 @@ func getRemoteLockBindKey(remoteService string, bind pb.LockTable) string { b.WriteString(bind.ServiceID) b.WriteByte('/') b.WriteString(strconv.FormatUint(bind.Version, 10)) + b.WriteByte('/') + b.WriteString(bind.AllocatorID) return b.String() } diff --git a/pkg/lockservice/service_test.go b/pkg/lockservice/service_test.go index da3ef13e4ee05..868d9a2187031 100644 --- a/pkg/lockservice/service_test.go +++ b/pkg/lockservice/service_test.go @@ -3545,6 +3545,120 @@ func TestKeepaliveOKFalseWithNewAllocatorVersionPurgesStaleBinds(t *testing.T) { }) } +func TestKeepaliveOKFalseFencesActiveTxn(t *testing.T) { + runLockServiceTests( + t, + []string{"s1"}, + func(alloc *lockTableAllocator, s []*service) { + l1 := s[0] + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + table := uint64(21511) + txnID := newTestTxnID(1) + option := newTestRowExclusiveOptions() + _, err := l1.Lock(ctx, table, newTestRows(1), txnID, option) + require.NoError(t, err) + require.NotNil(t, l1.tableGroups.get(0, table)) + + alloc.mu.Lock() + alloc.mu.services = make(map[string]*serviceBinds) + alloc.mu.lockTables = make(map[uint32]map[uint64]pb.LockTable) + alloc.allocatorID = "restarted-allocator-for-ok-false-fence" + alloc.version++ + alloc.mu.Unlock() + require.False(t, alloc.KeepLockTableBind(l1.serviceID)) + + l1.remote.keeper.(*lockTableKeeper).doKeepLockTableBind(ctx) + require.Nil(t, l1.tableGroups.get(0, table)) + + txn := l1.activeTxnHolder.getActiveTxn(txnID, false, "") + require.NotNil(t, txn) + txn.Lock() + require.True(t, txn.bindChanged) + txn.Unlock() + + _, err = l1.Lock(ctx, table, newTestRows(2), txnID, option) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrLockTableBindChanged)) + require.NoError(t, l1.Unlock(ctx, txnID, timestamp.Timestamp{})) + }, + ) +} + +func TestAllocatorObserverRejectsSupersededGetBindResponse(t *testing.T) { + runLockServiceTests( + t, + []string{"s1"}, + func(_ *lockTableAllocator, s []*service) { + l1 := s[0] + + oldAllocator := allocatorState{id: "old-allocator", version: 100} + newAllocator := allocatorState{id: "new-allocator", version: 90} + staleTable := uint64(21512) + + l1.allocatorVersionMu.Lock() + l1.lastAllocatorID = oldAllocator.id + l1.lastAllocatorVersion = oldAllocator.version + l1.allocatorVersionMu.Unlock() + + requestAllocator := l1.allocatorStateSnapshot() + _, accepted := l1.observeAllocatorStateWithHoldersFromSnapshot( + "accept-new-allocator-test", + newAllocator, + allocatorState{}, + false, + l1.tableGroups) + require.True(t, accepted) + require.Equal(t, newAllocator.id, l1.lastAllocatorID) + require.Equal(t, newAllocator.version, l1.lastAllocatorVersion) + + _, accepted = l1.observeAllocatorStateWithHoldersFromSnapshot( + "reject-stale-get-bind-test", + oldAllocator, + requestAllocator, + true, + l1.tableGroups) + require.False(t, accepted) + require.Equal(t, newAllocator.id, l1.lastAllocatorID) + require.Equal(t, newAllocator.version, l1.lastAllocatorVersion) + + if accepted { + staleBind := pb.LockTable{ + Group: 0, + Table: staleTable, + OriginTable: staleTable, + ServiceID: l1.serviceID, + Version: oldAllocator.version, + Valid: true, + AllocatorID: oldAllocator.id, + } + l1.tableGroups.set(0, staleTable, l1.createLockTableByBind(staleBind)) + } + require.Nil(t, l1.tableGroups.get(0, staleTable)) + }, + ) +} + +func TestAllocatorIDDistinguishesBindIdentity(t *testing.T) { + bindA := pb.LockTable{ + Group: 1, + Table: 2, + OriginTable: 3, + ServiceID: "s1", + Version: 4, + Sharding: pb.Sharding_ByRow, + AllocatorID: "allocator-a", + } + bindB := bindA + bindB.AllocatorID = "allocator-b" + + require.True(t, bindA.Changed(bindB)) + require.False(t, bindA.Equal(bindB)) + require.NotEqual(t, + getRemoteLockBindKey("remote-service", bindA), + getRemoteLockBindKey("remote-service", bindB)) +} + func TestAllocatorObserverPurgesRemoteAndProxyLockTables(t *testing.T) { runLockServiceTestsWithAdjustConfig( t, diff --git a/pkg/pb/lock/lock.go b/pkg/pb/lock/lock.go index 37aec878cb330..ae9866c3adcd9 100644 --- a/pkg/pb/lock/lock.go +++ b/pkg/pb/lock/lock.go @@ -111,7 +111,13 @@ func (m LockTable) Changed(v LockTable) bool { // Equal return true means same bind func (m LockTable) Equal(v LockTable) bool { - return m.Table == v.Table && m.Version == v.Version + return m.Group == v.Group && + m.Table == v.Table && + m.OriginTable == v.OriginTable && + m.ServiceID == v.ServiceID && + m.Version == v.Version && + m.Sharding == v.Sharding && + (m.AllocatorID == "" || v.AllocatorID == "" || m.AllocatorID == v.AllocatorID) } // DebugString returns the debug string