From abde6cb9c14fa32d5fc82b4aaded3fdfc5967726 Mon Sep 17 00:00:00 2001 From: denchick Date: Mon, 17 Mar 2025 14:38:12 +0100 Subject: [PATCH 1/7] refactor rulemgr.go --- pkg/config/router.go | 26 ------ pkg/config/rules.go | 29 ++++++ pkg/rulemgr/rulemgr.go | 157 +++++++++++++++++++------------- router/rulerouter/rulerouter.go | 37 +------- 4 files changed, 123 insertions(+), 126 deletions(-) create mode 100644 pkg/config/rules.go diff --git a/pkg/config/router.go b/pkg/config/router.go index ca75a0e3a..623a5c559 100644 --- a/pkg/config/router.go +++ b/pkg/config/router.go @@ -108,32 +108,6 @@ type QRouter struct { EnhancedMultiShardProcessing bool `json:"enhanced_multishard_processing" toml:"enhanced_multishard_processing" yaml:"enhanced_multishard_processing"` } -type BackendRule struct { - DB string `json:"db" yaml:"db" toml:"db"` - Usr string `json:"usr" yaml:"usr" toml:"usr"` - AuthRules map[string]*AuthBackendCfg `json:"auth_rules" yaml:"auth_rules" toml:"auth_rules"` - DefaultAuthRule *AuthBackendCfg `json:"auth_rule" yaml:"auth_rule" toml:"auth_rule"` - PoolDefault bool `json:"pool_default" yaml:"pool_default" toml:"pool_default"` - - ConnectionLimit int `json:"connection_limit" yaml:"connection_limit" toml:"connection_limit"` - ConnectionRetries int `json:"connection_retries" yaml:"connection_retries" toml:"connection_retries"` - ConnectionTimeout time.Duration `json:"connection_timeout" yaml:"connection_timeout" toml:"connection_timeout"` - KeepAlive time.Duration `json:"keep_alive" yaml:"keep_alive" toml:"keep_alive"` - TcpUserTimeout time.Duration `json:"tcp_user_timeout" yaml:"tcp_user_timeout" toml:"tcp_user_timeout"` -} - -type FrontendRule struct { - DB string `json:"db" yaml:"db" toml:"db"` - Usr string `json:"usr" yaml:"usr" toml:"usr"` - SearchPath string `json:"search_path" yaml:"search_path" toml:"search_path"` - AuthRule *AuthCfg `json:"auth_rule" yaml:"auth_rule" toml:"auth_rule"` - PoolMode PoolMode `json:"pool_mode" yaml:"pool_mode" toml:"pool_mode"` - PoolDiscard bool `json:"pool_discard" yaml:"pool_discard" toml:"pool_discard"` - PoolRollback bool `json:"pool_rollback" yaml:"pool_rollback" toml:"pool_rollback"` - PoolPreparedStatement bool `json:"pool_prepared_statement" yaml:"pool_prepared_statement" toml:"pool_prepared_statement"` - PoolDefault bool `json:"pool_default" yaml:"pool_default" toml:"pool_default"` -} - const ( TargetSessionAttrsRW = "read-write" TargetSessionAttrsRO = "read-only" diff --git a/pkg/config/rules.go b/pkg/config/rules.go new file mode 100644 index 000000000..a963148e9 --- /dev/null +++ b/pkg/config/rules.go @@ -0,0 +1,29 @@ +package config + +import "time" + +type BackendRule struct { + DB string `json:"db" yaml:"db" toml:"db"` + Usr string `json:"usr" yaml:"usr" toml:"usr"` + AuthRules map[string]*AuthBackendCfg `json:"auth_rules" yaml:"auth_rules" toml:"auth_rules"` + DefaultAuthRule *AuthBackendCfg `json:"auth_rule" yaml:"auth_rule" toml:"auth_rule"` + PoolDefault bool `json:"pool_default" yaml:"pool_default" toml:"pool_default"` + + ConnectionLimit int `json:"connection_limit" yaml:"connection_limit" toml:"connection_limit"` + ConnectionRetries int `json:"connection_retries" yaml:"connection_retries" toml:"connection_retries"` + ConnectionTimeout time.Duration `json:"connection_timeout" yaml:"connection_timeout" toml:"connection_timeout"` + KeepAlive time.Duration `json:"keep_alive" yaml:"keep_alive" toml:"keep_alive"` + TcpUserTimeout time.Duration `json:"tcp_user_timeout" yaml:"tcp_user_timeout" toml:"tcp_user_timeout"` +} + +type FrontendRule struct { + DB string `json:"db" yaml:"db" toml:"db"` + Usr string `json:"usr" yaml:"usr" toml:"usr"` + SearchPath string `json:"search_path" yaml:"search_path" toml:"search_path"` + AuthRule *AuthCfg `json:"auth_rule" yaml:"auth_rule" toml:"auth_rule"` + PoolMode PoolMode `json:"pool_mode" yaml:"pool_mode" toml:"pool_mode"` + PoolDiscard bool `json:"pool_discard" yaml:"pool_discard" toml:"pool_discard"` + PoolRollback bool `json:"pool_rollback" yaml:"pool_rollback" toml:"pool_rollback"` + PoolPreparedStatement bool `json:"pool_prepared_statement" yaml:"pool_prepared_statement" toml:"pool_prepared_statement"` + PoolDefault bool `json:"pool_default" yaml:"pool_default" toml:"pool_default"` +} diff --git a/pkg/rulemgr/rulemgr.go b/pkg/rulemgr/rulemgr.go index 521051954..66c56ebfd 100644 --- a/pkg/rulemgr/rulemgr.go +++ b/pkg/rulemgr/rulemgr.go @@ -16,11 +16,7 @@ type MatchMgr[T any] interface { type RulesMgr interface { MatchKeyFrontend(key route.Key) (*config.FrontendRule, error) MatchKeyBackend(key route.Key) (*config.BackendRule, error) - - Reload(frmp map[route.Key]*config.FrontendRule, - bemp map[route.Key]*config.BackendRule, - dfr *config.FrontendRule, - dbe *config.BackendRule) + Reload(frules []*config.FrontendRule, brules []*config.BackendRule) } type RulesMgrImpl struct { @@ -30,51 +26,52 @@ type RulesMgrImpl struct { } // TODO : unit tests -func (F *RulesMgrImpl) Reload(frmp map[route.Key]*config.FrontendRule, bemp map[route.Key]*config.BackendRule, dfr *config.FrontendRule, dbe *config.BackendRule) { +func (F *RulesMgrImpl) Reload(frules []*config.FrontendRule, brules []*config.BackendRule) { + mapFR, mapBE, defaultFR, defaultBE := parseRules(frules, brules) F.mu.Lock() defer F.mu.Unlock() fe := &MgrImpl[config.FrontendRule]{ - rule: frmp, + rule: mapFR, defaultRuleAllocator: func(key route.Key) *config.FrontendRule { - if dfr == nil { + if defaultFR == nil { return nil } spqrlog.Zero.Debug(). - Str("db", dfr.DB). - Str("user", dfr.Usr). + Str("db", defaultFR.DB). + Str("user", defaultFR.Usr). Msg("generating new dynamic rule") return &config.FrontendRule{ DB: key.DB(), Usr: key.Usr(), - SearchPath: dfr.SearchPath, - AuthRule: dfr.AuthRule, - PoolMode: dfr.PoolMode, - PoolDiscard: dfr.PoolDiscard, - PoolRollback: dfr.PoolRollback, - PoolPreparedStatement: dfr.PoolPreparedStatement, - PoolDefault: dfr.PoolDefault, + SearchPath: defaultFR.SearchPath, + AuthRule: defaultFR.AuthRule, + PoolMode: defaultFR.PoolMode, + PoolDiscard: defaultFR.PoolDiscard, + PoolRollback: defaultFR.PoolRollback, + PoolPreparedStatement: defaultFR.PoolPreparedStatement, + PoolDefault: defaultFR.PoolDefault, } }, } be := &MgrImpl[config.BackendRule]{ - rule: bemp, + rule: mapBE, defaultRuleAllocator: func(key route.Key) *config.BackendRule { - if dbe == nil { + if defaultBE == nil { return nil } return &config.BackendRule{ DB: key.DB(), Usr: key.Usr(), - AuthRules: dbe.AuthRules, - DefaultAuthRule: dbe.DefaultAuthRule, - PoolDefault: dbe.PoolDefault, - ConnectionLimit: dbe.ConnectionLimit, - ConnectionRetries: dbe.ConnectionRetries, - ConnectionTimeout: dbe.ConnectionTimeout, - KeepAlive: dbe.KeepAlive, - TcpUserTimeout: dbe.TcpUserTimeout, + AuthRules: defaultBE.AuthRules, + DefaultAuthRule: defaultBE.DefaultAuthRule, + PoolDefault: defaultBE.PoolDefault, + ConnectionLimit: defaultBE.ConnectionLimit, + ConnectionRetries: defaultBE.ConnectionRetries, + ConnectionTimeout: defaultBE.ConnectionTimeout, + KeepAlive: defaultBE.KeepAlive, + TcpUserTimeout: defaultBE.TcpUserTimeout, } }, } @@ -117,49 +114,79 @@ func (m *MgrImpl[T]) MatchKey(key route.Key, underlyingEntityName string) (*T, e " route for user:%s and db:%s is unconfigured in %s", key.Usr(), key.DB(), underlyingEntityName) } -func NewMgr(frmp map[route.Key]*config.FrontendRule, - bemp map[route.Key]*config.BackendRule, - dfr *config.FrontendRule, - dbe *config.BackendRule) RulesMgr { - fe := &MgrImpl[config.FrontendRule]{ - rule: frmp, - defaultRuleAllocator: func(key route.Key) *config.FrontendRule { - if dfr == nil { - return nil - } - // TODO add missing fields - return &config.FrontendRule{ - Usr: key.Usr(), - DB: key.DB(), - AuthRule: dfr.AuthRule, - PoolMode: dfr.PoolMode, - PoolPreparedStatement: dfr.PoolPreparedStatement, - } - }, +// TODO : unit tests +func parseRules(cfgFrontendRules []*config.FrontendRule, cfgBackendRules []*config.BackendRule) (map[route.Key]*config.FrontendRule, map[route.Key]*config.BackendRule, *config.FrontendRule, *config.BackendRule) { + frontendRules := map[route.Key]*config.FrontendRule{} + var defaultFrontendRule *config.FrontendRule + for _, frontendRule := range cfgFrontendRules { + if frontendRule.PoolDefault { + defaultFrontendRule = frontendRule + continue + } + spqrlog.Zero.Debug(). + Str("db", frontendRule.DB). + Str("user", frontendRule.Usr). + Msg("adding frontend rule") + key := *route.NewRouteKey(frontendRule.Usr, frontendRule.DB) + frontendRules[key] = frontendRule } - be := &MgrImpl[config.BackendRule]{ - rule: bemp, - defaultRuleAllocator: func(key route.Key) *config.BackendRule { - if dbe == nil { - return nil - } - // TODO add missing fields - return &config.BackendRule{ - Usr: key.Usr(), - DB: key.DB(), - AuthRules: dbe.AuthRules, - DefaultAuthRule: dbe.DefaultAuthRule, - ConnectionLimit: dbe.ConnectionLimit, - ConnectionRetries: dbe.ConnectionRetries, - ConnectionTimeout: dbe.ConnectionTimeout, - } - }, + backendRules := map[route.Key]*config.BackendRule{} + var defaultBackendRule *config.BackendRule + for _, backendRule := range cfgBackendRules { + if backendRule.PoolDefault { + defaultBackendRule = backendRule + continue + } + key := *route.NewRouteKey(backendRule.Usr, backendRule.DB) + backendRules[key] = backendRule } + return frontendRules, backendRules, defaultFrontendRule, defaultBackendRule +} + +// func NewMgr(frmp map[route.Key]*config.FrontendRule, +// +// bemp map[route.Key]*config.BackendRule, +// dfr *config.FrontendRule, +// dbe *config.BackendRule) RulesMgr { +func NewMgr(frules []*config.FrontendRule, brules []*config.BackendRule) RulesMgr { + frmp, bemp, dfr, dbr := parseRules(frules, brules) return &RulesMgrImpl{ - fe: fe, - be: be, + fe: &MgrImpl[config.FrontendRule]{ + rule: frmp, + defaultRuleAllocator: func(key route.Key) *config.FrontendRule { + if dfr == nil { + return nil + } + // TODO add missing fields + return &config.FrontendRule{ + Usr: key.Usr(), + DB: key.DB(), + AuthRule: dfr.AuthRule, + PoolMode: dfr.PoolMode, + PoolPreparedStatement: dfr.PoolPreparedStatement, + } + }, + }, + be: &MgrImpl[config.BackendRule]{ + rule: bemp, + defaultRuleAllocator: func(key route.Key) *config.BackendRule { + if dbr == nil { + return nil + } + // TODO add missing fields + return &config.BackendRule{ + Usr: key.Usr(), + DB: key.DB(), + AuthRules: dbr.AuthRules, + DefaultAuthRule: dbr.DefaultAuthRule, + ConnectionLimit: dbr.ConnectionLimit, + ConnectionRetries: dbr.ConnectionRetries, + ConnectionTimeout: dbr.ConnectionTimeout, + } + }, + }, } } diff --git a/router/rulerouter/rulerouter.go b/router/rulerouter/rulerouter.go index 92fd5f46c..619111e0a 100644 --- a/router/rulerouter/rulerouter.go +++ b/router/rulerouter/rulerouter.go @@ -95,37 +95,6 @@ func (r *RuleRouterImpl) ForEach(cb func(sh shard.Shardinfo) error) error { return r.routePool.ForEach(cb) } -// TODO : unit tests -func ParseRules(rcfg *config.Router) (map[route.Key]*config.FrontendRule, map[route.Key]*config.BackendRule, *config.FrontendRule, *config.BackendRule) { - frontendRules := map[route.Key]*config.FrontendRule{} - var defaultFrontendRule *config.FrontendRule - for _, frontendRule := range rcfg.FrontendRules { - if frontendRule.PoolDefault { - defaultFrontendRule = frontendRule - continue - } - spqrlog.Zero.Debug(). - Str("db", frontendRule.DB). - Str("user", frontendRule.Usr). - Msg("adding frontend rule") - key := *route.NewRouteKey(frontendRule.Usr, frontendRule.DB) - frontendRules[key] = frontendRule - } - - backendRules := map[route.Key]*config.BackendRule{} - var defaultBackendRule *config.BackendRule - for _, backendRule := range rcfg.BackendRules { - if backendRule.PoolDefault { - defaultBackendRule = backendRule - continue - } - key := *route.NewRouteKey(backendRule.Usr, backendRule.DB) - backendRules[key] = backendRule - } - - return frontendRules, backendRules, defaultFrontendRule, defaultBackendRule -} - // TODO : unit tests func (r *RuleRouterImpl) Reload(configPath string) error { /* @@ -153,8 +122,7 @@ func (r *RuleRouterImpl) Reload(configPath string) error { spqrlog.ReloadLogger(rcfg.LogFileName, rcfg.LogLevel, rcfg.PrettyLogging) - frontendRules, backendRules, defaultFrontendRule, defaultBackendRule := ParseRules(rcfg) - r.rmgr.Reload(frontendRules, backendRules, defaultFrontendRule, defaultBackendRule) + r.rmgr.Reload(rcfg.FrontendRules, rcfg.BackendRules) if r.notifier != nil { if err = r.notifier.Ready(); err != nil { @@ -177,11 +145,10 @@ func (r *RuleRouterImpl) Reload(configPath string) error { } func NewRouter(tlsconfig *tls.Config, rcfg *config.Router, notifier *notifier.Notifier) *RuleRouterImpl { - frontendRules, backendRules, defaultFrontendRule, defaultBackendRule := ParseRules(rcfg) return &RuleRouterImpl{ routePool: NewRouterPoolImpl(rcfg.ShardMapping), rcfg: rcfg, - rmgr: rulemgr.NewMgr(frontendRules, backendRules, defaultFrontendRule, defaultBackendRule), + rmgr: rulemgr.NewMgr(rcfg.FrontendRules, rcfg.BackendRules), tlsconfig: tlsconfig, clmp: sync.Map{}, notifier: notifier, From a86e07d1e27188f58d31809ee50b369948ad50fd Mon Sep 17 00:00:00 2001 From: denchick Date: Mon, 17 Mar 2025 14:39:10 +0100 Subject: [PATCH 2/7] add pkg/rulemgr/rulemgr_test.go --- pkg/rulemgr/rulemgr.go | 1 - pkg/rulemgr/rulemgr_test.go | 133 ++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 pkg/rulemgr/rulemgr_test.go diff --git a/pkg/rulemgr/rulemgr.go b/pkg/rulemgr/rulemgr.go index 66c56ebfd..3552a9295 100644 --- a/pkg/rulemgr/rulemgr.go +++ b/pkg/rulemgr/rulemgr.go @@ -114,7 +114,6 @@ func (m *MgrImpl[T]) MatchKey(key route.Key, underlyingEntityName string) (*T, e " route for user:%s and db:%s is unconfigured in %s", key.Usr(), key.DB(), underlyingEntityName) } -// TODO : unit tests func parseRules(cfgFrontendRules []*config.FrontendRule, cfgBackendRules []*config.BackendRule) (map[route.Key]*config.FrontendRule, map[route.Key]*config.BackendRule, *config.FrontendRule, *config.BackendRule) { frontendRules := map[route.Key]*config.FrontendRule{} var defaultFrontendRule *config.FrontendRule diff --git a/pkg/rulemgr/rulemgr_test.go b/pkg/rulemgr/rulemgr_test.go new file mode 100644 index 000000000..8c2c077fa --- /dev/null +++ b/pkg/rulemgr/rulemgr_test.go @@ -0,0 +1,133 @@ +package rulemgr + +import ( + "testing" + + "github.com/pg-sharding/spqr/pkg/config" + "github.com/pg-sharding/spqr/router/route" + "github.com/stretchr/testify/assert" +) + +func TestParseRules(t *testing.T) { + tests := []struct { + name string + frontendRules []*config.FrontendRule + backendRules []*config.BackendRule + wantFrontendRules map[route.Key]*config.FrontendRule + wantBackendRules map[route.Key]*config.BackendRule + wantDefaultFR *config.FrontendRule + wantDefaultBR *config.BackendRule + }{ + { + name: "empty rules", + frontendRules: []*config.FrontendRule{}, + backendRules: []*config.BackendRule{}, + wantFrontendRules: map[route.Key]*config.FrontendRule{}, + wantBackendRules: map[route.Key]*config.BackendRule{}, + wantDefaultFR: nil, + wantDefaultBR: nil, + }, + { + name: "single rules with defaults", + frontendRules: []*config.FrontendRule{ + { + Usr: "user1", + DB: "db1", + PoolDefault: false, + }, + { + Usr: "default", + DB: "default", + PoolDefault: true, + }, + }, + backendRules: []*config.BackendRule{ + { + Usr: "user1", + DB: "db1", + PoolDefault: false, + }, + { + Usr: "default", + DB: "default", + PoolDefault: true, + }, + }, + wantFrontendRules: map[route.Key]*config.FrontendRule{ + *route.NewRouteKey("user1", "db1"): { + Usr: "user1", + DB: "db1", + PoolDefault: false, + }, + }, + wantBackendRules: map[route.Key]*config.BackendRule{ + *route.NewRouteKey("user1", "db1"): { + Usr: "user1", + DB: "db1", + PoolDefault: false, + }, + }, + wantDefaultFR: &config.FrontendRule{ + Usr: "default", + DB: "default", + PoolDefault: true, + }, + wantDefaultBR: &config.BackendRule{ + Usr: "default", + DB: "default", + PoolDefault: true, + }, + }, + { + name: "only frontend rules exist", + frontendRules: []*config.FrontendRule{ + { + Usr: "user1", + DB: "db1", + PoolDefault: false, + }, + { + Usr: "user2", + DB: "db2", + PoolDefault: false, + }, + { + Usr: "default", + DB: "default", + PoolDefault: true, + }, + }, + backendRules: []*config.BackendRule{}, + wantFrontendRules: map[route.Key]*config.FrontendRule{ + *route.NewRouteKey("user1", "db1"): { + Usr: "user1", + DB: "db1", + PoolDefault: false, + }, + *route.NewRouteKey("user2", "db2"): { + Usr: "user2", + DB: "db2", + PoolDefault: false, + }, + }, + wantBackendRules: map[route.Key]*config.BackendRule{}, + wantDefaultFR: &config.FrontendRule{ + Usr: "default", + DB: "default", + PoolDefault: true, + }, + wantDefaultBR: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFR, gotBR, gotDefaultFR, gotDefaultBR := parseRules(tt.frontendRules, tt.backendRules) + + assert.Equal(t, tt.wantFrontendRules, gotFR) + assert.Equal(t, tt.wantBackendRules, gotBR) + assert.Equal(t, tt.wantDefaultFR, gotDefaultFR) + assert.Equal(t, tt.wantDefaultBR, gotDefaultBR) + }) + } +} From 034b08bc227ed45dc12feed9e707a95b9bdf2f15 Mon Sep 17 00:00:00 2001 From: denchick Date: Mon, 17 Mar 2025 14:55:11 +0100 Subject: [PATCH 3/7] use rulemgr.RulesMgr in coordinator --- coordinator/provider/coordinator.go | 28 ++++++++++++++++++---------- pkg/config/coordinator.go | 28 ++++++++++++++-------------- pkg/conn/instance.go | 2 -- pkg/rulemgr/rulemgr_test.go | 9 +++++++++ 4 files changed, 41 insertions(+), 26 deletions(-) diff --git a/coordinator/provider/coordinator.go b/coordinator/provider/coordinator.go index 9dd505000..b9f6e0239 100644 --- a/coordinator/provider/coordinator.go +++ b/coordinator/provider/coordinator.go @@ -29,6 +29,7 @@ import ( "github.com/pg-sharding/spqr/pkg/models/topology" "github.com/pg-sharding/spqr/pkg/pool" routerproto "github.com/pg-sharding/spqr/pkg/protos" + "github.com/pg-sharding/spqr/pkg/rulemgr" "github.com/pg-sharding/spqr/pkg/shard" "github.com/pg-sharding/spqr/pkg/spqrlog" "github.com/pg-sharding/spqr/qdb" @@ -196,6 +197,7 @@ func DialRouter(r *topology.Router) (*grpc.ClientConn, error) { const defaultWatchRouterTimeout = time.Second type qdbCoordinator struct { + rmgr rulemgr.RulesMgr tlsconfig *tls.Config db qdb.XQDB cache *cache.SchemaCache @@ -314,6 +316,7 @@ func NewCoordinator(tlsconfig *tls.Config, db qdb.XQDB) (*qdbCoordinator, error) return &qdbCoordinator{ db: db, tlsconfig: tlsconfig, + rmgr: rulemgr.NewMgr(config.CoordinatorConfig().FrontendRules, []*config.BackendRule{}), }, nil } @@ -1876,19 +1879,24 @@ func (qc *qdbCoordinator) PrepareClient(nconn net.Conn, pt port.RouterPortType) Bool("ssl", tlsconfig != nil). Msg("init client connection OK") - var authRule *config.AuthCfg - if config.CoordinatorConfig().Auth != nil { - authRule = config.CoordinatorConfig().Auth - } else { - spqrlog.Zero.Warn().Msg("ATTENTION! Skipping auth checking!") - authRule = &config.AuthCfg{ - Method: config.AuthOK, + // match client to frontend rule + key := *route.NewRouteKey(cl.Usr(), cl.DB()) + frRule, err := qc.rmgr.MatchKeyFrontend(key) + if err != nil { + for _, msg := range []pgproto3.BackendMessage{ + &pgproto3.ErrorResponse{ + Severity: "ERROR", + Message: err.Error(), + }, + } { + if err := cl.Send(msg); err != nil { + return nil, fmt.Errorf("failed to make route failure response: %w", err) + } } + return nil, err } - if err := cl.AssignRule(&config.FrontendRule{ - AuthRule: authRule, - }); err != nil { + if err := cl.AssignRule(frRule); err != nil { return nil, err } diff --git a/pkg/config/coordinator.go b/pkg/config/coordinator.go index 95e3e4159..0eb7a3db4 100644 --- a/pkg/config/coordinator.go +++ b/pkg/config/coordinator.go @@ -14,20 +14,20 @@ import ( var cfgCoordinator Coordinator type Coordinator struct { - LogLevel string `json:"log_level" toml:"log_level" yaml:"log_level"` - PrettyLogging bool `json:"pretty_logging" toml:"pretty_logging" yaml:"pretty_logging"` - QdbAddr string `json:"qdb_addr" toml:"qdb_addr" yaml:"qdb_addr"` - CoordinatorPort string `json:"coordinator_port" toml:"coordinator_port" yaml:"coordinator_port"` - GrpcApiPort string `json:"grpc_api_port" toml:"grpc_api_port" yaml:"grpc_api_port"` - Host string `json:"host" toml:"host" yaml:"host"` - Auth *AuthCfg `json:"auth" toml:"auth" yaml:"auth"` - FrontendTLS *TLSConfig `json:"frontend_tls" yaml:"frontend_tls" toml:"frontend_tls"` - ShardDataCfg string `json:"shard_data" toml:"shard_data" yaml:"shard_data"` - UseSystemdNotifier bool `json:"use_systemd_notifier" toml:"use_systemd_notifier" yaml:"use_systemd_notifier"` - SystemdNotifierDebug bool `json:"systemd_notifier_debug" toml:"systemd_notifier_debug" yaml:"systemd_notifier_debug"` - IterationTimeout time.Duration `json:"iteration_timeout" toml:"iteration_timeout" yaml:"iteration_timeout"` - EnableRoleSystem bool `json:"enable_role_system" toml:"enable_role_system" yaml:"enable_role_system"` - RolesFile string `json:"roles_file" toml:"roles_file" yaml:"roles_file"` + LogLevel string `json:"log_level" toml:"log_level" yaml:"log_level"` + PrettyLogging bool `json:"pretty_logging" toml:"pretty_logging" yaml:"pretty_logging"` + QdbAddr string `json:"qdb_addr" toml:"qdb_addr" yaml:"qdb_addr"` + CoordinatorPort string `json:"coordinator_port" toml:"coordinator_port" yaml:"coordinator_port"` + GrpcApiPort string `json:"grpc_api_port" toml:"grpc_api_port" yaml:"grpc_api_port"` + Host string `json:"host" toml:"host" yaml:"host"` + FrontendTLS *TLSConfig `json:"frontend_tls" yaml:"frontend_tls" toml:"frontend_tls"` + FrontendRules []*FrontendRule `json:"frontend_rules" toml:"frontend_rules" yaml:"frontend_rules"` + ShardDataCfg string `json:"shard_data" toml:"shard_data" yaml:"shard_data"` + UseSystemdNotifier bool `json:"use_systemd_notifier" toml:"use_systemd_notifier" yaml:"use_systemd_notifier"` + SystemdNotifierDebug bool `json:"systemd_notifier_debug" toml:"systemd_notifier_debug" yaml:"systemd_notifier_debug"` + IterationTimeout time.Duration `json:"iteration_timeout" toml:"iteration_timeout" yaml:"iteration_timeout"` + EnableRoleSystem bool `json:"enable_role_system" toml:"enable_role_system" yaml:"enable_role_system"` + RolesFile string `json:"roles_file" toml:"roles_file" yaml:"roles_file"` } // LoadCoordinatorCfg loads the coordinator configuration from the specified file path. diff --git a/pkg/conn/instance.go b/pkg/conn/instance.go index 856944113..69f6352ba 100644 --- a/pkg/conn/instance.go +++ b/pkg/conn/instance.go @@ -187,8 +187,6 @@ func setTCPUserTimeout(d time.Duration) func(string, string, syscall.RawConn) er #define TCP_USER_TIMEOUT 18 // How long for loss retry before timeout */ - sysErr = syscall.SetsockoptInt(int(fd), syscall.SOL_TCP, 0x12, - int(d.Milliseconds())) }) if sysErr != nil { return os.NewSyscallError("setsockopt", sysErr) diff --git a/pkg/rulemgr/rulemgr_test.go b/pkg/rulemgr/rulemgr_test.go index 8c2c077fa..cdc15216b 100644 --- a/pkg/rulemgr/rulemgr_test.go +++ b/pkg/rulemgr/rulemgr_test.go @@ -27,6 +27,15 @@ func TestParseRules(t *testing.T) { wantDefaultFR: nil, wantDefaultBR: nil, }, + { + name: "nil rules", + frontendRules: nil, + backendRules: nil, + wantFrontendRules: map[route.Key]*config.FrontendRule{}, + wantBackendRules: map[route.Key]*config.BackendRule{}, + wantDefaultFR: nil, + wantDefaultBR: nil, + }, { name: "single rules with defaults", frontendRules: []*config.FrontendRule{ From ea4a2f25bae1c419ccef7e6fefb1fc515e9a30d9 Mon Sep 17 00:00:00 2001 From: denchick Date: Mon, 17 Mar 2025 14:55:29 +0100 Subject: [PATCH 4/7] yaml development --- test/feature/conf/coordinator.yaml | 9 ++++++--- test/feature/conf/coordinator2.yaml | 9 ++++++--- test/feature/conf/coordinator_three_shards.yaml | 9 ++++++--- test/feature/conf/router_coordinator.yaml | 9 ++++++--- test/feature/conf/router_coordinator_2.yaml | 9 ++++++--- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/test/feature/conf/coordinator.yaml b/test/feature/conf/coordinator.yaml index 0290aba65..41ed0be9a 100644 --- a/test/feature/conf/coordinator.yaml +++ b/test/feature/conf/coordinator.yaml @@ -4,6 +4,9 @@ grpc_api_port: 7003 qdb_addr: '[regress_qdb_0_1]:2379' log_level: info shard_data: '/spqr/test/feature/conf/shard_data.yaml' -auth: - auth_method: clear_text - password: password +frontend_rules: + - db: regress + usr: regress + auth_rule: + auth_method: clear_text + password: password diff --git a/test/feature/conf/coordinator2.yaml b/test/feature/conf/coordinator2.yaml index 3e61def6d..04fcc1c0a 100644 --- a/test/feature/conf/coordinator2.yaml +++ b/test/feature/conf/coordinator2.yaml @@ -4,6 +4,9 @@ grpc_api_port: 7003 qdb_addr: '[regress_qdb_0_1]:2379' log_level: INFO shard_data: '/spqr/test/feature/conf/shard_data.yaml' -auth: - auth_method: clear_text - password: password +frontend_rules: + - db: regress + usr: regress + auth_rule: + auth_method: clear_text + password: password diff --git a/test/feature/conf/coordinator_three_shards.yaml b/test/feature/conf/coordinator_three_shards.yaml index d8b31d385..9a66f3911 100644 --- a/test/feature/conf/coordinator_three_shards.yaml +++ b/test/feature/conf/coordinator_three_shards.yaml @@ -4,6 +4,9 @@ grpc_api_port: 7003 qdb_addr: '[regress_qdb_0_1]:2379' log_level: info shard_data: '/spqr/test/feature/conf/shard_data_three_shards.yaml' -auth: - auth_method: clear_text - password: password +frontend_rules: + - db: regress + usr: regress + auth_rule: + auth_method: clear_text + password: password diff --git a/test/feature/conf/router_coordinator.yaml b/test/feature/conf/router_coordinator.yaml index cb3abc977..ce8d23a4d 100644 --- a/test/feature/conf/router_coordinator.yaml +++ b/test/feature/conf/router_coordinator.yaml @@ -2,6 +2,9 @@ host: '' coordinator_port: 7002 grpc_api_port: 7003 qdb_addr: '[regress_qdb_0_1]:2379' -auth: - auth_method: clear_text - password: password +frontend_rules: + - db: regress + usr: regress + auth_rule: + auth_method: clear_text + password: password diff --git a/test/feature/conf/router_coordinator_2.yaml b/test/feature/conf/router_coordinator_2.yaml index cb3abc977..b8e0503b9 100644 --- a/test/feature/conf/router_coordinator_2.yaml +++ b/test/feature/conf/router_coordinator_2.yaml @@ -2,6 +2,9 @@ host: '' coordinator_port: 7002 grpc_api_port: 7003 qdb_addr: '[regress_qdb_0_1]:2379' -auth: - auth_method: clear_text - password: password +frontend_rules: + - db: regress + usr: regress + auth_rule: + auth_method: clear_text + password: password \ No newline at end of file From 0405747ce0deef9007de5be17905808a5dce581f Mon Sep 17 00:00:00 2001 From: denchick Date: Mon, 17 Mar 2025 15:08:11 +0100 Subject: [PATCH 5/7] minor fix --- pkg/config/rules.go | 8 +++++--- pkg/conn/instance.go | 2 ++ pkg/rulemgr/rulemgr.go | 5 ----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/config/rules.go b/pkg/config/rules.go index a963148e9..8f915bd8f 100644 --- a/pkg/config/rules.go +++ b/pkg/config/rules.go @@ -17,10 +17,12 @@ type BackendRule struct { } type FrontendRule struct { - DB string `json:"db" yaml:"db" toml:"db"` - Usr string `json:"usr" yaml:"usr" toml:"usr"` + DB string `json:"db" yaml:"db" toml:"db"` + Usr string `json:"usr" yaml:"usr" toml:"usr"` + AuthRule *AuthCfg `json:"auth_rule" yaml:"auth_rule" toml:"auth_rule"` + + // Pool settings and search_path does not take effect for coordinator SearchPath string `json:"search_path" yaml:"search_path" toml:"search_path"` - AuthRule *AuthCfg `json:"auth_rule" yaml:"auth_rule" toml:"auth_rule"` PoolMode PoolMode `json:"pool_mode" yaml:"pool_mode" toml:"pool_mode"` PoolDiscard bool `json:"pool_discard" yaml:"pool_discard" toml:"pool_discard"` PoolRollback bool `json:"pool_rollback" yaml:"pool_rollback" toml:"pool_rollback"` diff --git a/pkg/conn/instance.go b/pkg/conn/instance.go index 69f6352ba..856944113 100644 --- a/pkg/conn/instance.go +++ b/pkg/conn/instance.go @@ -187,6 +187,8 @@ func setTCPUserTimeout(d time.Duration) func(string, string, syscall.RawConn) er #define TCP_USER_TIMEOUT 18 // How long for loss retry before timeout */ + sysErr = syscall.SetsockoptInt(int(fd), syscall.SOL_TCP, 0x12, + int(d.Milliseconds())) }) if sysErr != nil { return os.NewSyscallError("setsockopt", sysErr) diff --git a/pkg/rulemgr/rulemgr.go b/pkg/rulemgr/rulemgr.go index 3552a9295..6573d6b67 100644 --- a/pkg/rulemgr/rulemgr.go +++ b/pkg/rulemgr/rulemgr.go @@ -144,11 +144,6 @@ func parseRules(cfgFrontendRules []*config.FrontendRule, cfgBackendRules []*conf return frontendRules, backendRules, defaultFrontendRule, defaultBackendRule } -// func NewMgr(frmp map[route.Key]*config.FrontendRule, -// -// bemp map[route.Key]*config.BackendRule, -// dfr *config.FrontendRule, -// dbe *config.BackendRule) RulesMgr { func NewMgr(frules []*config.FrontendRule, brules []*config.BackendRule) RulesMgr { frmp, bemp, dfr, dbr := parseRules(frules, brules) return &RulesMgrImpl{ From 6bd5fa3158aaf6e9f2fb30aace0dfbd98636be85 Mon Sep 17 00:00:00 2001 From: denchick Date: Mon, 17 Mar 2025 16:02:45 +0100 Subject: [PATCH 6/7] fix regress --- test/regress/conf/coordinator.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/regress/conf/coordinator.yaml b/test/regress/conf/coordinator.yaml index 2bd778ee4..ab4071331 100644 --- a/test/regress/conf/coordinator.yaml +++ b/test/regress/conf/coordinator.yaml @@ -4,3 +4,8 @@ coordinator_port: 7002 grpc_api_port: 7003 qdb_addr: '[regress_qdb_0_1]:2379' shard_data: '/spqr/test/feature/conf/shard_data.yaml' +frontend_rules: + - db: regress + usr: regress + auth_rule: + auth_method: ok From d58af664e4b303c85bd7a45b751fe570c3c8fbe0 Mon Sep 17 00:00:00 2001 From: denchick Date: Mon, 17 Mar 2025 17:16:32 +0100 Subject: [PATCH 7/7] add documentation --- .github/actions/spelling/expect.txt | 2 ++ docs/configuration/coordinator.mdx | 38 +++++++++++++++++++---------- docs/configuration/router.mdx | 2 +- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index fbf9c063f..e21de2352 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -79,6 +79,7 @@ datname dbe dbi dbpool +dbr dbtpcc dcd ddl @@ -140,6 +141,7 @@ FQRN frmp frrule frsm +frules gcflags getconn gettime diff --git a/docs/configuration/coordinator.mdx b/docs/configuration/coordinator.mdx index 32afa26e9..e66228584 100644 --- a/docs/configuration/coordinator.mdx +++ b/docs/configuration/coordinator.mdx @@ -12,18 +12,30 @@ Refer to the [pkg/config/coordinator.go](https://github.com/pg-sharding/spqr/blo ## Coordinator Settings -| Setting | Description | Possible Values | -|--------------------------|--------------------------------------------------------------------|-----------------------| +| Setting | Description | Possible Values | +|--------------------------|--------------------------------------------------------------------|------------------------| | `log_level` | The level of logging output. | `debug`, `info`, `warning`, `error`, `fatal`| -| `pretty_logging` | Whether to write logs in an colorized, human-friendly format. | `true`, `false` | -| `qdb_addr` | the address of the QDB server | Any valid address | -| `host` | The host address the coordinator listens on. | Any valid hostname | -| `coordinator_port` | The port number for the coordinator. | Any valid port number | -| `grpc_api_port` | The port number for the gRPC API. | Any valid port number | -| `auth` | See [auth.mdx](./auth) | Object of `AuthCfg` | -| `frontend_tls` | See [auth.mdx](./auth) | Object of `TLSConfig` | -| `use_systemd_notifier` | Whether to use systemd notifier. | `true`, `false` | -| `systemd_notifier_debug` | Whether to run systemd notifier in debug mode. | `true`, `false` | -| `enable_role_system` | Whether to enable the [role-based access control system](./roles). | `true`, `false` | -| `roles_file` | The file path to the [roles](./roles) configuration. | Any valid file path | +| `pretty_logging` | Whether to write logs in an colorized, human-friendly format. | `true`, `false` | +| `qdb_addr` | the address of the QDB server | Any valid address | +| `host` | The host address the coordinator listens on. | Any valid hostname | +| `coordinator_port` | The port number for the coordinator. | Any valid port number | +| `grpc_api_port` | The port number for the gRPC API. | Any valid port number | +| `auth` | See [auth.mdx](./auth) | Object of `AuthCfg` | +| `frontend_tls` | See [auth.mdx](./auth) | Object of `TLSConfig` | +| `frontend_rules` | The rules for frontend connections. | List of `FrontendRule` | +| `use_systemd_notifier` | Whether to use systemd notifier. | `true`, `false` | +| `systemd_notifier_debug` | Whether to run systemd notifier in debug mode. | `true`, `false` | +| `enable_role_system` | Whether to enable the [role-based access control system](./roles). | `true`, `false` | +| `roles_file` | The file path to the [roles](./roles) configuration. | Any valid file path | +## Frontend Rules + +Frontend rule is a specification of how clients connect to the admin console. + +Refer to the `FrontendRule` struct in the [pkg/config/rules.go](https://github.com/pg-sharding/spqr/blob/master/pkg/config/rules.go) file for the most up-to-date configuration options. + +| Setting | Description | Possible Values | +|---------------------------|--------------------------------------------------------------------------------|--------------------------| +| `db` | The database name to which the rule applies | Any valid database name | +| `usr` | The user name for which the rule is applicable | Any valid username | +| `auth_rule` | See [General Auth Settings](./auth.go) | Object of `AuthCfg` | diff --git a/docs/configuration/router.mdx b/docs/configuration/router.mdx index fc75b75cc..ded4720c8 100644 --- a/docs/configuration/router.mdx +++ b/docs/configuration/router.mdx @@ -38,7 +38,7 @@ Refer to the [pkg/config/router.go](https://github.com/pg-sharding/spqr/blob/mas Frontend rule is a specification of how clients connect to the router. -Refer to the `FrontendRule` struct in the [pkg/config/router.go](https://github.com/pg-sharding/spqr/blob/master/pkg/config/router.go) file for the most up-to-date configuration options. +Refer to the `FrontendRule` struct in the [pkg/config/rules.go](https://github.com/pg-sharding/spqr/blob/master/pkg/config/rules.go) file for the most up-to-date configuration options. | Setting | Description | Possible Values | |---------------------------|--------------------------------------------------------------------------------|--------------------------|