fix gold pipeline loader

Author: svilupp

src/pipeline_defaults.jl

@@ -40,9 +40,6 @@ let MODEL_CHAT = MODEL_CHAT, MODEL_EMBEDDING = MODEL_EMBEDDING,
             generator_kwargs = (;
                 answerer_kwargs = (;
                     model = MODEL_CHAT),
-                embedder_kwargs = (;
-                    truncate_dimension = EMBEDDING_DIMENSION,
-                    model = MODEL_EMBEDDING),
                 refiner_kwargs = (;
                     model = MODEL_CHAT))))
     ## Silver - reranking added
@@ -65,9 +62,6 @@ let MODEL_CHAT = MODEL_CHAT, MODEL_EMBEDDING = MODEL_EMBEDDING,
             generator_kwargs = (;
                 answerer_kwargs = (;
                     model = MODEL_CHAT),
-                embedder_kwargs = (;
-                    truncate_dimension = EMBEDDING_DIMENSION,
-                    model = MODEL_EMBEDDING),
                 refiner_kwargs = (;
                     model = MODEL_CHAT))))
     ## Gold  - reranking + web-search
@@ -84,14 +78,13 @@ let MODEL_CHAT = MODEL_CHAT, MODEL_EMBEDDING = MODEL_EMBEDDING,
                 rephraser_kwargs = (;
                     model = MODEL_CHAT),
                 embedder_kwargs = (;
+                    truncate_dimension = nothing,
                     model = MODEL_EMBEDDING),
                 tagger_kwargs = (;
                     model = MODEL_CHAT)),
             generator_kwargs = (;
                 answerer_kwargs = (;
                     model = MODEL_CHAT),
-                embedder_kwargs = (;
-                    model = MODEL_EMBEDDING),
                 refiner_kwargs = (;
                     model = MODEL_CHAT))))
 end
@@ -119,7 +112,7 @@ See available pipeline options via `keys(RAG_CONFIGURATIONS)`.
 
 Logic:
 - Updates the global `MODEL_CHAT` and `MODEL_EMBEDDING` to the requested models.
-- Update the global `EMBEDDING_DIMENSION` for the requested embedding dimensionality after truncation (`embedding_dimension`).
+- Updates the global `EMBEDDING_DIMENSION` for the requested embedding dimensionality after truncation (`embedding_dimension`).
 - Updates the global `RAG_CONFIG` and `RAG_KWARGS` to the requested `option`.
 - Updates the global `LOADED_CONFIG_KEY` to the configuration key for the given `option` and `kwargs` (used by the artifact system to download the correct knowledge packs).
 
@@ -139,31 +132,34 @@ update_pipeline!(:bronze; model_chat = "llama3", model_embedding="nomic-embed-te
 load_index!()
 ```
 """
-function update_pipeline!(option::Symbol = :bronze; model_chat = MODEL_CHAT,
-        model_embedding = MODEL_EMBEDDING, verbose::Bool = true,
-        embedding_dimension::Integer = EMBEDDING_DIMENSION)
+function update_pipeline!(
+        option::Symbol = :bronze; model_chat::Union{String, Nothing} = nothing,
+        model_embedding::Union{String, Nothing} = nothing, verbose::Bool = true,
+        embedding_dimension::Union{Integer, Nothing} = nothing)
     global RAG_CONFIGURATIONS, RAG_CONFIG, RAG_KWARGS, MODEL_CHAT, MODEL_EMBEDDING, EMBEDDING_DIMENSION, LOADED_CONFIG_KEY
 
+    ## Set from globals if not provided
+    model_chat = !isnothing(model_chat) ? model_chat : MODEL_CHAT
+    model_embedding = !isnothing(model_embedding) ? model_embedding : MODEL_EMBEDDING
+    ## Do not set embedding dimensions, we might need to extract it from kwargs
+
     ## WARN about limited support for nomic-embed-text -- we need to create repeatable process
     if model_embedding == "nomic-embed-text"
         @warn "Knowledge packs for `nomic-embed-text` are currently not built automatically, so they might be missing / outdated. Please switch to OpenAI `text-embedding-3-large` for the best experience."
     end
 
     @assert haskey(RAG_CONFIGURATIONS, option) "Invalid option: $option. Select one of: $(join(keys(RAG_CONFIGURATIONS),", "))"
-    @assert embedding_dimension in [0, 1024, 3072] "Invalid embedding_dimension: $(embedding_dimension). Supported: 0, 1024, 3072. See the available artifacts."
+    @assert (isnothing(embedding_dimension)||embedding_dimension in [0, 1024]) "Invalid embedding_dimension: $(embedding_dimension). Supported: 0 (no truncation) or 1024. See the available artifacts."
     ## Model-specific checks, they do not fail but at least warn
-    if model_embedding == "nomic-embed-text" && !iszero(embedding_dimension)
+    if model_embedding == "nomic-embed-text" &&
+       !(iszero(embedding_dimension) || isnothing(embedding_dimension))
         @warn "Invalid configuration for knowledge packs! For `nomic-embed-text`, `embedding_dimension` must be 0. See the available artifacts."
     end
-    if model_embedding == "text-embedding-3-large" && embedding_dimension ∉ [1024, 0]
+    if model_embedding == "text-embedding-3-large" &&
+       (embedding_dimension ∉ [1024, 0] || !isnothing(embedding_dimension))
         @warn "Invalid configuration for knowledge packs! For `text-embedding-3-large`, `embedding_dimension` must be 0 or 1024. See the available artifacts."
     end
 
-    ## Update model references
-    MODEL_CHAT = model_chat
-    MODEL_EMBEDDING = model_embedding
-    EMBEDDING_DIMENSION = embedding_dimension
-
     config = RAG_CONFIGURATIONS[option][:config]
     kwargs = RAG_CONFIGURATIONS[option][:kwargs]
     # update models in kwargs to the ones requested
@@ -179,8 +175,18 @@ function update_pipeline!(option::Symbol = :bronze; model_chat = MODEL_CHAT,
     if !isnothing(embedding_dimension)
         kwargs = setpropertynested(
             kwargs, [:embedder_kwargs], :truncate_dimension, embedding_dimension)
+    else
+        ## load the value from defaults in config -- to match the artifacts
+        val = getpropertynested(kwargs, [:embedder_kwargs], :truncate_dimension, nothing)
+        embedding_dimension = isnothing(val) ? 0 : val
     end
 
+    ## Update GLOBAL variables
+    MODEL_CHAT = model_chat
+    MODEL_EMBEDDING = model_embedding
+    @info embedding_dimension
+    EMBEDDING_DIMENSION = embedding_dimension
+
     ## Set the options
     config_key = get_config_key(config, kwargs)
     ## detect significant changes