Data race between Lifecycler and memberlist join

[dimitarvdimitrov]

A test failed in Mimir because of a data race in the ring and memberlist packages. This is running dskit commit b9a439d and Mimir at around grafana/mimir@9bb0752

09:50:33 mimir-1: ==================
09:50:33 mimir-1: WARNING: DATA RACE
09:50:33 mimir-1: Write at 0x00c0008aaf30 by goroutine 221:
09:50:33 mimir-1: runtime.mapassign_faststr()
09:50:33 mimir-1: /usr/local/go/src/runtime/map_faststr.go:203 +0x0
09:50:33 mimir-1: github.com/grafana/dskit/ring.(*Desc).mergeWithTime()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/model.go:229 +0xd9a
09:50:33 mimir-1: github.com/grafana/dskit/ring.(*Desc).Merge()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/model.go:195 +0x64
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.computeNewValue()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/memberlist_client.go:1426 +0x68
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KV).mergeValueForKey()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/memberlist_client.go:1377 +0x28a
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KV).mergeBytesValueForKey()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/memberlist_client.go:1358 +0x1a6
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KV).MergeRemoteState()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/memberlist_client.go:1319 +0x484
09:50:33 mimir-1: github.com/hashicorp/memberlist.(*Memberlist).mergeRemoteState()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/hashicorp/memberlist/net.go:1261 +0x25c
09:50:33 mimir-1: github.com/hashicorp/memberlist.(*Memberlist).pushPullNode()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/hashicorp/memberlist/state.go:664 +0x16c
09:50:33 mimir-1: github.com/hashicorp/memberlist.(*Memberlist).Join()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/hashicorp/memberlist/memberlist.go:275 +0x507
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KV).joinMembersBatch()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/memberlist_client.go:678 +0xc4
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KV).joinMembersInBatches()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/memberlist_client.go:655 +0x127
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KV).joinMembersWithRetries()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/memberlist_client.go:600 +0x2e4
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KV).joinMembersOnStartup()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/memberlist_client.go:575 +0x44b
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KV).running()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/memberlist_client.go:476 +0x6a
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KV).running-fm()
09:50:33 mimir-1: <autogenerated>:1 +0x47
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).main()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x398
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).StartAsync.func1.2()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x33
09:50:33 mimir-1: Previous read at 0x00c0008aaf30 by goroutine 3028:
09:50:33 mimir-1: runtime.mapiterinit()
09:50:33 mimir-1: /usr/local/go/src/runtime/map.go:816 +0x0
09:50:33 mimir-1: github.com/grafana/dskit/ring.(*Lifecycler).updateCounters()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/lifecycler.go:920 +0x126
09:50:33 mimir-1: github.com/grafana/dskit/ring.(*Lifecycler).initRing()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/lifecycler.go:712 +0x7fc
09:50:33 mimir-1: github.com/grafana/dskit/ring.(*Lifecycler).loop()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/lifecycler.go:465 +0x68
09:50:33 mimir-1: github.com/grafana/dskit/ring.(*Lifecycler).loop-fm()
09:50:33 mimir-1: <autogenerated>:1 +0x47
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).main()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x398
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).StartAsync.func1.2()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x33
09:50:33 mimir-1: Goroutine 221 (running) created at:
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).StartAsync.func1()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dc
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).switchState()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:139 +0x12a
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).StartAsync()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:116 +0xb7
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KVInitService).GetMemberlistKV.func1()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/kv_init_service.go:52 +0x201
09:50:33 mimir-1: sync.(*Once).doSlow()
09:50:33 mimir-1: /usr/local/go/src/sync/once.go:74 +0xf0
09:50:33 mimir-1: sync.(*Once).Do()
09:50:33 mimir-1: /usr/local/go/src/sync/once.go:65 +0x44
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KVInitService).GetMemberlistKV()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/memberlist/kv_init_service.go:49 +0x70
09:50:33 mimir-1: github.com/grafana/dskit/kv/memberlist.(*KVInitService).GetMemberlistKV-fm()
09:50:33 mimir-1: <autogenerated>:1 +0x33
09:50:33 mimir-1: github.com/grafana/dskit/kv.createClient()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/client.go:158 +0x532
09:50:33 mimir-1: github.com/grafana/dskit/kv.NewClient()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/client.go:134 +0x157
09:50:33 mimir-1: github.com/grafana/dskit/ring.New()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:215 +0x2dc
09:50:33 mimir-1: github.com/grafana/mimir/pkg/mimir.(*Mimir).initIngesterRing()
09:50:33 mimir-1: /__w/mimir/mimir/pkg/mimir/modules.go:342 +0x224
09:50:33 mimir-1: github.com/grafana/mimir/pkg/mimir.(*Mimir).initIngesterRing-fm()
09:50:33 mimir-1: <autogenerated>:1 +0x33
09:50:33 mimir-1: github.com/grafana/dskit/modules.(*Manager).initModule()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/modules/modules.go:136 +0x35a
09:50:33 mimir-1: github.com/grafana/dskit/modules.(*Manager).InitModuleServices()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/modules/modules.go:108 +0x136
09:50:33 mimir-1: github.com/grafana/mimir/pkg/mimir.(*Mimir).Run()
09:50:33 mimir-1: /__w/mimir/mimir/pkg/mimir/mimir.go:821 +0x6a7
09:50:33 mimir-1: main.main()
09:50:33 mimir-1: /__w/mimir/mimir/cmd/mimir/main.go:227 +0x1d76
09:50:33 mimir-1: Goroutine 3028 (running) created at:
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).StartAsync.func1()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dc
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).switchState()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:139 +0x12a
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).StartAsync()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:116 +0xb7
09:50:33 mimir-1: github.com/grafana/mimir/pkg/ingester.(*Ingester).starting()
09:50:33 mimir-1: /__w/mimir/mimir/pkg/ingester/ingester.go:497 +0x4e4
09:50:33 mimir-1: github.com/grafana/mimir/pkg/ingester.(*Ingester).starting-fm()
09:50:33 mimir-1: <autogenerated>:1 +0x47
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).main()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:157 +0xe6
09:50:33 mimir-1: github.com/grafana/dskit/services.(*BasicService).StartAsync.func1.2()
09:50:33 mimir-1: /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x33
09:50:33 mimir-1: ==================