From ef60ac3e75c4d0fcf018d75477e29c12804c350d Mon Sep 17 00:00:00 2001 From: Hamid Reza Ranjbar Date: Tue, 28 May 2024 10:22:27 +0330 Subject: [PATCH 1/7] feat: mapping slice of complex struct #298 --- env.go | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++++ env_test.go | 41 ++++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/env.go b/env.go index d4bdf55..82cf9c6 100644 --- a/env.go +++ b/env.go @@ -168,6 +168,19 @@ func customOptions(opt Options) Options { return opt } +func optionsWithSliceEnvPrefix(opts Options, index int) Options { + return Options{ + Environment: opts.Environment, + TagName: opts.TagName, + RequiredIfNoDef: opts.RequiredIfNoDef, + OnSet: opts.OnSet, + Prefix: fmt.Sprintf("%s%d_", opts.Prefix, index), + UseFieldNameByDefault: opts.UseFieldNameByDefault, + FuncMap: opts.FuncMap, + rawEnvVars: opts.rawEnvVars, + } +} + func optionsWithEnvPrefix(field reflect.StructField, opts Options) Options { return Options{ Environment: opts.Environment, @@ -313,9 +326,102 @@ func doParseField(refField reflect.Value, refTypeField reflect.StructField, proc return doParse(refField, processField, optionsWithEnvPrefix(refTypeField, opts)) } + if isSliceOfStructs(refTypeField, opts) { + err := doParseSlice(refField, processField, optionsWithEnvPrefix(refTypeField, opts)) + if len(err) > 0 { + return err[0] + } + } + return nil } +func isSliceOfStructs(refTypeField reflect.StructField, opts Options) bool { + field := refTypeField.Type + if reflect.Ptr == field.Kind() { + field = field.Elem() + } + + if reflect.Slice != field.Kind() { + return false + } + + field = field.Elem() + + if reflect.Ptr == field.Kind() { + field = field.Elem() + } + + _, ignore := defaultBuiltInParsers[field.Kind()] + + if !ignore { + _, ignore = opts.FuncMap[field] + } + + if !ignore { + _, ignore = reflect.New(field).Interface().(encoding.TextUnmarshaler) + } + + if !ignore { + ignore = reflect.Struct != field.Kind() + } + return !ignore +} + +func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) []error { + if !strings.HasSuffix(opts.Prefix, string(underscore)) { + opts.Prefix += string(underscore) + } + + var environments []string + for environment, _ := range opts.Environment { + if strings.HasPrefix(environment, opts.Prefix) { + environments = append(environments, environment) + } + } + + var errors []error + if len(environments) > 0 { + counter := 0 + for finished := false; !finished; { + finished = true + prefix := fmt.Sprintf("%s%d%c", opts.Prefix, counter, underscore) + for _, variable := range environments { + if strings.HasPrefix(variable, prefix) { + counter++ + finished = false + break + } + } + } + + sliceType := ref.Type() + if reflect.Ptr == ref.Kind() { + sliceType = sliceType.Elem() + } + + result := reflect.MakeSlice(sliceType, counter, counter) + + for i := 0; i < counter; i++ { + iRef := result.Index(i) + err := doParse(iRef, processField, optionsWithSliceEnvPrefix(opts, i)) + if err != nil { + errors = append(errors, err) + } + } + + if reflect.Ptr == ref.Kind() { + resultPtr := reflect.New(sliceType) + resultPtr.Elem().Set(result) + result = resultPtr + } + + ref.Set(result) + } + + return errors +} + func setField(refField reflect.Value, refTypeField reflect.StructField, opts Options, fieldParams FieldParams) error { value, err := get(fieldParams, opts) if err != nil { diff --git a/env_test.go b/env_test.go index f322e10..220e43a 100644 --- a/env_test.go +++ b/env_test.go @@ -2077,3 +2077,44 @@ func TestIssue308(t *testing.T) { isNoErr(t, Parse(&cfg)) isEqual(t, Issue308Map{"FOO": []string{"BAR", "ZAZ"}}, cfg.Inner) } + +func TestIssue298(t *testing.T) { + type Test struct { + Str string `env:"STR"` + Num int `env:"NUM"` + } + type ComplexConfig struct { + Foo *[]Test `envPrefix:"FOO_"` + Bar []Test `envPrefix:"BAR_"` + Baz *Test + } + + t.Setenv("FOO_0_STR", "f0t") + t.Setenv("FOO_0_NUM", "101") + t.Setenv("FOO_1_STR", "f1t") + t.Setenv("FOO_1_NUM", "111") + + t.Setenv("BAR_0_STR", "b0t") + t.Setenv("BAR_0_NUM", "202") + t.Setenv("BAR_1_STR", "b1t") + t.Setenv("BAR_1_NUM", "212") + + t.Setenv("STR", "bt") + t.Setenv("NUM", "10") + + cfg := ComplexConfig{} + isNoErr(t, Parse(&cfg)) + + isEqual(t, "f0t", (*cfg.Foo)[0].Str) + isEqual(t, 101, (*cfg.Foo)[0].Num) + isEqual(t, "f1t", (*cfg.Foo)[1].Str) + isEqual(t, 111, (*cfg.Foo)[1].Num) + + isEqual(t, "b0t", cfg.Bar[0].Str) + isEqual(t, 202, cfg.Bar[0].Num) + isEqual(t, "b1t", cfg.Bar[1].Str) + isEqual(t, 212, cfg.Bar[1].Num) + + isEqual(t, "bt", cfg.Baz.Str) + isEqual(t, 10, cfg.Baz.Num) +} From 2db80ccf7ed740551c7035f2265b2dda16609329 Mon Sep 17 00:00:00 2001 From: Hamid Reza Ranjbar Date: Tue, 28 May 2024 15:38:10 +0330 Subject: [PATCH 2/7] feat: support predefined values and pre initialized structs --- env.go | 22 ++++++++++++++++------ env_test.go | 10 +++++++--- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/env.go b/env.go index 82cf9c6..fc4282b 100644 --- a/env.go +++ b/env.go @@ -396,15 +396,25 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) } sliceType := ref.Type() + var initialized int if reflect.Ptr == ref.Kind() { sliceType = sliceType.Elem() + // Due to the rest of code the pre-initialized slice has no chance to arrive here + initialized = 0 + } else { + initialized = ref.Len() } - - result := reflect.MakeSlice(sliceType, counter, counter) - - for i := 0; i < counter; i++ { - iRef := result.Index(i) - err := doParse(iRef, processField, optionsWithSliceEnvPrefix(opts, i)) + var capacity int + if capacity = counter; initialized > counter { + capacity = initialized + } + result := reflect.MakeSlice(sliceType, capacity, capacity) + for i := 0; i < capacity; i++ { + item := result.Index(i) + if i < initialized { + item.Set(ref.Index(i)) + } + err := doParse(item, processField, optionsWithSliceEnvPrefix(opts, i)) if err != nil { errors = append(errors, err) } diff --git a/env_test.go b/env_test.go index 220e43a..fad20d6 100644 --- a/env_test.go +++ b/env_test.go @@ -2095,14 +2095,18 @@ func TestIssue298(t *testing.T) { t.Setenv("FOO_1_NUM", "111") t.Setenv("BAR_0_STR", "b0t") - t.Setenv("BAR_0_NUM", "202") + //t.Setenv("BAR_0_NUM", "202") // Not overridden t.Setenv("BAR_1_STR", "b1t") t.Setenv("BAR_1_NUM", "212") t.Setenv("STR", "bt") t.Setenv("NUM", "10") - cfg := ComplexConfig{} + sample := make([]Test, 1) + sample[0].Str = "overridden text" + sample[0].Num = 99999999 + cfg := ComplexConfig{Bar: sample} + isNoErr(t, Parse(&cfg)) isEqual(t, "f0t", (*cfg.Foo)[0].Str) @@ -2111,7 +2115,7 @@ func TestIssue298(t *testing.T) { isEqual(t, 111, (*cfg.Foo)[1].Num) isEqual(t, "b0t", cfg.Bar[0].Str) - isEqual(t, 202, cfg.Bar[0].Num) + isEqual(t, 99999999, cfg.Bar[0].Num) isEqual(t, "b1t", cfg.Bar[1].Str) isEqual(t, 212, cfg.Bar[1].Num) From 96950158924862a35d71db9153090a92a3e22369 Mon Sep 17 00:00:00 2001 From: Hamid Reza Ranjbar Date: Sun, 2 Jun 2024 21:18:20 +0330 Subject: [PATCH 3/7] refactor: some improvement --- env.go | 24 ++++++++++-------------- env_test.go | 2 +- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/env.go b/env.go index fc4282b..96ddcce 100644 --- a/env.go +++ b/env.go @@ -2,6 +2,7 @@ package env import ( "encoding" + "errors" "fmt" "net/url" "os" @@ -327,10 +328,7 @@ func doParseField(refField reflect.Value, refTypeField reflect.StructField, proc } if isSliceOfStructs(refTypeField, opts) { - err := doParseSlice(refField, processField, optionsWithEnvPrefix(refTypeField, opts)) - if len(err) > 0 { - return err[0] - } + return doParseSlice(refField, processField, optionsWithEnvPrefix(refTypeField, opts)) } return nil @@ -368,7 +366,7 @@ func isSliceOfStructs(refTypeField reflect.StructField, opts Options) bool { return !ignore } -func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) []error { +func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) error { if !strings.HasSuffix(opts.Prefix, string(underscore)) { opts.Prefix += string(underscore) } @@ -380,7 +378,6 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) } } - var errors []error if len(environments) > 0 { counter := 0 for finished := false; !finished; { @@ -405,19 +402,17 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) initialized = ref.Len() } var capacity int - if capacity = counter; initialized > counter { - capacity = initialized + if capacity = initialized; counter > initialized { + capacity = counter } + var errorList = make([]error, capacity) result := reflect.MakeSlice(sliceType, capacity, capacity) for i := 0; i < capacity; i++ { item := result.Index(i) if i < initialized { item.Set(ref.Index(i)) } - err := doParse(item, processField, optionsWithSliceEnvPrefix(opts, i)) - if err != nil { - errors = append(errors, err) - } + errorList[i] = doParse(item, processField, optionsWithSliceEnvPrefix(opts, i)) } if reflect.Ptr == ref.Kind() { @@ -425,11 +420,12 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) resultPtr.Elem().Set(result) result = resultPtr } - ref.Set(result) + + return errors.Join(errorList...) } - return errors + return nil } func setField(refField reflect.Value, refTypeField reflect.StructField, opts Options, fieldParams FieldParams) error { diff --git a/env_test.go b/env_test.go index fad20d6..544fc83 100644 --- a/env_test.go +++ b/env_test.go @@ -2085,7 +2085,7 @@ func TestIssue298(t *testing.T) { } type ComplexConfig struct { Foo *[]Test `envPrefix:"FOO_"` - Bar []Test `envPrefix:"BAR_"` + Bar []Test `envPrefix:"BAR"` Baz *Test } From 8369621a2a17f00751ca74fadafbd106fc9aab42 Mon Sep 17 00:00:00 2001 From: Hamid Reza Ranjbar Date: Mon, 3 Jun 2024 21:54:21 +0330 Subject: [PATCH 4/7] refactor: some improvement --- env.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/env.go b/env.go index 96ddcce..e90b172 100644 --- a/env.go +++ b/env.go @@ -378,6 +378,7 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) } } + var errorList []error if len(environments) > 0 { counter := 0 for finished := false; !finished; { @@ -396,16 +397,18 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) var initialized int if reflect.Ptr == ref.Kind() { sliceType = sliceType.Elem() - // Due to the rest of code the pre-initialized slice has no chance to arrive here + // Due to the rest of code the pre-initialized slice has no chance for this situation initialized = 0 } else { initialized = ref.Len() } + //capacity := int(math.Max(float64(initialized), float64(counter))) + var capacity int if capacity = initialized; counter > initialized { capacity = counter } - var errorList = make([]error, capacity) + errorList = make([]error, capacity) result := reflect.MakeSlice(sliceType, capacity, capacity) for i := 0; i < capacity; i++ { item := result.Index(i) @@ -421,11 +424,9 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) result = resultPtr } ref.Set(result) - - return errors.Join(errorList...) } - return nil + return errors.Join(errorList...) } func setField(refField reflect.Value, refTypeField reflect.StructField, opts Options, fieldParams FieldParams) error { From 6ea60ec8b5ed53160aeeb68c450d0dae7beedfa5 Mon Sep 17 00:00:00 2001 From: Hamid Reza Ranjbar Date: Mon, 3 Jun 2024 22:51:38 +0330 Subject: [PATCH 5/7] test: support normal features for nested fields --- env.go | 10 ++++------ env_test.go | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/env.go b/env.go index e90b172..112ea2d 100644 --- a/env.go +++ b/env.go @@ -2,7 +2,6 @@ package env import ( "encoding" - "errors" "fmt" "net/url" "os" @@ -378,7 +377,6 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) } } - var errorList []error if len(environments) > 0 { counter := 0 for finished := false; !finished; { @@ -402,20 +400,20 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) } else { initialized = ref.Len() } - //capacity := int(math.Max(float64(initialized), float64(counter))) var capacity int if capacity = initialized; counter > initialized { capacity = counter } - errorList = make([]error, capacity) result := reflect.MakeSlice(sliceType, capacity, capacity) for i := 0; i < capacity; i++ { item := result.Index(i) if i < initialized { item.Set(ref.Index(i)) } - errorList[i] = doParse(item, processField, optionsWithSliceEnvPrefix(opts, i)) + if err := doParse(item, processField, optionsWithSliceEnvPrefix(opts, i)); err != nil { + return err + } } if reflect.Ptr == ref.Kind() { @@ -426,7 +424,7 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) ref.Set(result) } - return errors.Join(errorList...) + return nil } func setField(refField reflect.Value, refTypeField reflect.StructField, opts Options, fieldParams FieldParams) error { diff --git a/env_test.go b/env_test.go index 544fc83..b75d78d 100644 --- a/env_test.go +++ b/env_test.go @@ -2122,3 +2122,20 @@ func TestIssue298(t *testing.T) { isEqual(t, "bt", cfg.Baz.Str) isEqual(t, 10, cfg.Baz.Num) } + +func TestIssue298ErrorNestedFieldRequiredNotSet(t *testing.T) { + type Test struct { + Str string `env:"STR,required"` + Num int `env:"NUM"` + } + type ComplexConfig struct { + Foo *[]Test `envPrefix:"FOO"` + } + + t.Setenv("FOO_0_NUM", "101") + + cfg := ComplexConfig{} + err := Parse(&cfg) + isErrorWithMessage(t, err, `env: required environment variable "FOO_0_STR" is not set`) + isTrue(t, errors.Is(err, EnvVarIsNotSetError{})) +} From 89473ac6da728a028d759b9e6154f30799bb9d68 Mon Sep 17 00:00:00 2001 From: Hamid Reza Ranjbar Date: Mon, 3 Jun 2024 23:26:49 +0330 Subject: [PATCH 6/7] test: trying to fix `gofumpt` lint issues --- env.go | 2 +- env_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/env.go b/env.go index 112ea2d..9d584c3 100644 --- a/env.go +++ b/env.go @@ -371,7 +371,7 @@ func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) } var environments []string - for environment, _ := range opts.Environment { + for environment := range opts.Environment { if strings.HasPrefix(environment, opts.Prefix) { environments = append(environments, environment) } diff --git a/env_test.go b/env_test.go index b75d78d..459c8dc 100644 --- a/env_test.go +++ b/env_test.go @@ -2095,7 +2095,7 @@ func TestIssue298(t *testing.T) { t.Setenv("FOO_1_NUM", "111") t.Setenv("BAR_0_STR", "b0t") - //t.Setenv("BAR_0_NUM", "202") // Not overridden + // t.Setenv("BAR_0_NUM", "202") // Not overridden t.Setenv("BAR_1_STR", "b1t") t.Setenv("BAR_1_NUM", "212") From ece55fb5d0c363002b5e7cd57f29a459fbd4fe31 Mon Sep 17 00:00:00 2001 From: Hamid Reza Ranjbar Date: Tue, 16 Jul 2024 00:52:36 +0330 Subject: [PATCH 7/7] chore: add sample for complex struct in readme --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++ env.go | 2 +- env_test.go | 12 +++++++----- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9199fe9..a3f27b3 100644 --- a/README.md +++ b/README.md @@ -476,6 +476,61 @@ func main() { } ``` +### Complex objects inside array (slice) + +You can set sub-struct field values inside a slice by naming the environment variables with sequential numbers starting from 0 (without omitting numbers in between) and an underscore. +It is possible to use prefix tag too. + +Here's an example with and without prefix tag: + +```go +package main + +import ( + "fmt" + "log" + + "github.com/caarlos0/env/v11" +) + +type Test struct { + Str string `env:"STR"` + Num int `env:"NUM"` +} +type ComplexConfig struct { + Baz []Test `env:",init"` + Bar []Test `envPrefix:"BAR"` + Foo *[]Test `envPrefix:"FOO_"` +} + +func main() { + cfg := &ComplexConfig{} + opts := env.Options{ + Environment: map[string]string{ + "0_STR": "bt", + "1_NUM": "10", + + "FOO_0_STR": "b0t", + "FOO_1_STR": "b1t", + "FOO_1_NUM": "212", + + "BAR_0_STR": "f0t", + "BAR_0_NUM": "101", + "BAR_1_STR": "f1t", + "BAR_1_NUM": "111", + }, + } + + // Load env vars. + if err := env.ParseWithOptions(cfg, opts); err != nil { + log.Fatal(err) + } + + // Print the loaded data. + fmt.Printf("%+v\n", cfg) +} +``` + ### On set hooks You might want to listen to value sets and, for example, log something or do diff --git a/env.go b/env.go index 8f270f9..9585321 100644 --- a/env.go +++ b/env.go @@ -366,7 +366,7 @@ func isSliceOfStructs(refTypeField reflect.StructField, opts Options) bool { } func doParseSlice(ref reflect.Value, processField processFieldFn, opts Options) error { - if !strings.HasSuffix(opts.Prefix, string(underscore)) { + if opts.Prefix != "" && !strings.HasSuffix(opts.Prefix, string(underscore)) { opts.Prefix += string(underscore) } diff --git a/env_test.go b/env_test.go index 2d0bce6..a24e2e1 100644 --- a/env_test.go +++ b/env_test.go @@ -2160,7 +2160,7 @@ func TestIssue298(t *testing.T) { type ComplexConfig struct { Foo *[]Test `envPrefix:"FOO_"` Bar []Test `envPrefix:"BAR"` - Baz *Test + Baz []Test `env:",init"` } t.Setenv("FOO_0_STR", "f0t") @@ -2173,8 +2173,8 @@ func TestIssue298(t *testing.T) { t.Setenv("BAR_1_STR", "b1t") t.Setenv("BAR_1_NUM", "212") - t.Setenv("STR", "bt") - t.Setenv("NUM", "10") + t.Setenv("0_STR", "bt") + t.Setenv("1_NUM", "10") sample := make([]Test, 1) sample[0].Str = "overridden text" @@ -2193,8 +2193,10 @@ func TestIssue298(t *testing.T) { isEqual(t, "b1t", cfg.Bar[1].Str) isEqual(t, 212, cfg.Bar[1].Num) - isEqual(t, "bt", cfg.Baz.Str) - isEqual(t, 10, cfg.Baz.Num) + isEqual(t, "bt", cfg.Baz[0].Str) + isEqual(t, 0, cfg.Baz[0].Num) + isEqual(t, "", cfg.Baz[1].Str) + isEqual(t, 10, cfg.Baz[1].Num) } func TestIssue298ErrorNestedFieldRequiredNotSet(t *testing.T) {