WIP: Current redesign to make configuration management easier + speci…

…fication of scope and catalog.

docs/source/config.rst

@@ -12,6 +12,7 @@ be read from there.
 
 .. autopydantic_model:: agentc_core.config.config.RemoteCatalogConfig
 .. autopydantic_model:: agentc_core.config.config.LocalCatalogConfig
+.. autopydantic_model:: agentc_core.config.config.EmbeddingModelConfig
 .. autopydantic_model:: agentc_core.config.config.CommandLineConfig
 .. autopydantic_model:: agentc_core.config.config.VersioningConfig
 .. autopydantic_model:: agentc_core.config.config.Config

libs/agentc_cli/agentc_cli/cmds/add.py

@@ -8,6 +8,7 @@
 import subprocess
 import typing
 
+from .util import logging_command
 from agentc_core.config import Config
 from agentc_core.record.descriptor import RecordKind
 
@@ -149,6 +150,7 @@ def add_sqlpp_query(output: pathlib.Path, template_env: jinja2.Environment):
     subprocess.run([default_editor, f"{output_file}"])
 
 
+@logging_command(parent_logger=logger)
 def cmd_add(
     cfg: Config = None,
     *,

libs/agentc_cli/agentc_cli/cmds/clean.py

@@ -7,6 +7,7 @@
 import shutil
 import typing
 
+from .util import logging_command
 from agentc_core.config import Config
 from agentc_core.defaults import DEFAULT_ACTIVITY_FOLDER
 from agentc_core.defaults import DEFAULT_AUDIT_SCOPE
@@ -103,6 +104,7 @@ def clean_db(
     return len(all_errs)
 
 
+@logging_command(parent_logger=logger)
 def cmd_clean(
     cfg: Config = None,
     *,

libs/agentc_cli/agentc_cli/cmds/env.py

@@ -1,11 +1,16 @@
 import click
 import importlib.util
 import json
+import logging
 import re
 
+from .util import logging_command
 from agentc_core.config import Config
 
+logger = logging.getLogger(__name__)
 
+
+@logging_command(logger)
 def cmd_env(cfg: Config = None):
     if cfg is None:
         cfg = Config()

libs/agentc_cli/agentc_cli/cmds/execute.py

@@ -10,6 +10,7 @@
 from .util import DASHES
 from .util import KIND_COLORS
 from .util import get_catalog
+from agentc_cli.cmds.util import logging_command
 from agentc_core.config import Config
 from agentc_core.provider import ToolProvider
 from agentc_core.record.descriptor import RecordDescriptor
@@ -27,6 +28,7 @@
 logger = logging.getLogger(__name__)
 
 
+@logging_command(logger)
 def cmd_execute(
     cfg: Config = None,
     *,
@@ -58,14 +60,7 @@ def cmd_execute(
         raise ValueError("Either local FS or DB catalog must be specified!")
 
     # Initialize a catalog instance.
-    catalog = get_catalog(
-        catalog_path=cfg.CatalogPath(),
-        bucket=cfg.bucket,
-        cluster=cfg.Cluster() if with_db else None,
-        force=force,
-        include_dirty=include_dirty,
-        kind="tool",
-    )
+    catalog = get_catalog(cfg=cfg, force=force, include_dirty=include_dirty, kind="tool")
 
     # create temp directory for code dump
     _dir = cfg.codegen_output if cfg.codegen_output is not None else os.getcwd()

libs/agentc_cli/agentc_cli/cmds/find.py

@@ -1,5 +1,4 @@
 import click
-import couchbase.cluster
 import logging
 import pydantic
 import textwrap
@@ -8,6 +7,7 @@
 from .util import DASHES
 from .util import KIND_COLORS
 from .util import get_catalog
+from .util import logging_command
 from agentc_core.annotation import AnnotationPredicate
 from agentc_core.catalog import SearchResult
 from agentc_core.config import Config
@@ -41,6 +41,7 @@ def check_one_field_populated(cls, values):
         return values
 
 
+@logging_command(logger)
 def cmd_find(
     cfg: Config = None,
     *,
@@ -84,15 +85,7 @@ def cmd_find(
         raise ValueError("Either local FS or DB catalog must be specified!")
 
     # Execute the find on our catalog.
-    cluster: couchbase.cluster.Cluster = cfg.Cluster()
-    catalog = get_catalog(
-        catalog_path=cfg.CatalogPath(),
-        bucket=cfg.bucket,
-        cluster=cluster,
-        force=force,
-        include_dirty=include_dirty,
-        kind=kind,
-    )
+    catalog = get_catalog(cfg=cfg, force=force, include_dirty=include_dirty, kind=kind)
     annotations_predicate = AnnotationPredicate(annotations) if annotations is not None else None
     search_results = [
         SearchResult(entry=x.entry, delta=x.delta)

libs/agentc_cli/agentc_cli/cmds/index.py

@@ -9,12 +9,12 @@
 from ..cmds.util import load_repository
 from .util import DASHES
 from .util import KIND_COLORS
+from .util import logging_command
 from agentc_core.catalog import __version__ as CATALOG_SCHEMA_VERSION
 from agentc_core.catalog.index import MetaVersion
 from agentc_core.catalog.index import index_catalog
 from agentc_core.catalog.version import lib_version
 from agentc_core.config import Config
-from agentc_core.defaults import DEFAULT_EMBEDDING_MODEL
 from agentc_core.defaults import DEFAULT_MAX_ERRS
 from agentc_core.defaults import DEFAULT_MODEL_INPUT_CATALOG_FILE
 from agentc_core.defaults import DEFAULT_SCAN_DIRECTORY_OPTS
@@ -25,12 +25,12 @@
 logger = logging.getLogger(__name__)
 
 
+@logging_command(logger)
 def cmd_index(
     cfg: Config = None,
     *,
     source_dirs: list[str | os.PathLike],
     kinds: list[typing.Literal["tool", "model-input"]],
-    embedding_model_name: str = DEFAULT_EMBEDDING_MODEL,
     dry_run: bool = False,
 ):
     assert all(k in {"tool", "model-input"} for k in kinds)
@@ -52,7 +52,10 @@ def cmd_index(
     # approach of opening & reading file contents directly,
     repo, get_path_version = load_repository(pathlib.Path(os.getcwd()))
     embedding_model = EmbeddingModel(
-        embedding_model_name=embedding_model_name,
+        embedding_model_name=cfg.embedding_model_name,
+        embedding_model_url=cfg.embedding_model_url,
+        embedding_model_auth=cfg.embedding_model_auth,
+        sentence_transformers_model_cache=cfg.sentence_transformers_model_cache,
         catalog_path=cfg.CatalogPath(),
     )
 

libs/agentc_cli/agentc_cli/cmds/init.py

@@ -9,6 +9,7 @@
 import typing
 
 from .util import CATALOG_KINDS
+from .util import logging_command
 from agentc_core.analytics.create import create_analytics_views
 from agentc_core.analytics.create import create_query_udfs
 from agentc_core.config import Config
@@ -27,6 +28,7 @@
 logger = logging.getLogger(__name__)
 
 
+@logging_command(parent_logger=logger)
 def cmd_init(
     cfg: Config = None,
     *,
@@ -38,12 +40,14 @@ def cmd_init(
         cfg = Config()
 
     if local:
+        logger.debug("Initializing local-FS catalog and activity.")
         if "catalog" in targets:
             init_local_catalog(cfg)
         if "activity" in targets:
             init_local_activity(cfg)
 
     if db:
+        logger.debug("Initializing DB catalog and activity.")
         cluster = cfg.Cluster()
         if "catalog" in targets:
             init_db_catalog(cfg, cluster)
@@ -112,6 +116,13 @@ def init_db_catalog(cfg: Config, cluster: couchbase.cluster.Cluster):
     # ---------------------------------------------------------------------------------------- #
     #                               GSI and Vector Indexes                                     #
     # ---------------------------------------------------------------------------------------- #
+    embedding_model = EmbeddingModel(
+        embedding_model_name=cfg.embedding_model_name,
+        embedding_model_url=cfg.embedding_model_url,
+        embedding_model_auth=cfg.embedding_model_auth,
+        sentence_transformers_model_cache=cfg.sentence_transformers_model_cache,
+    )
+    dims = len(embedding_model.encode("test"))
     for kind in CATALOG_KINDS:
         click.secho(f"Now building the GSI indexes for the {kind} catalog.", fg="yellow")
         completion_status, err = create_gsi_indexes(cfg, kind, True)
@@ -121,8 +132,6 @@ def init_db_catalog(cfg: Config, cluster: couchbase.cluster.Cluster):
             click.secho(f"All GSI indexes for the {kind} catalog have been successfully created!\n", fg="green")
 
         click.secho(f"Now building the vector index for the {kind} catalog.", fg="yellow")
-        embedding_model = EmbeddingModel(embedding_model_name=cfg.embedding_model)
-        dims = len(embedding_model.encode("test"))
         _, err = create_vector_index(
             cfg=cfg,
             scope=DEFAULT_CATALOG_SCOPE,

libs/agentc_cli/agentc_cli/cmds/ls.py

@@ -1,13 +1,18 @@
 import click
+import logging
 import typing
 
 from agentc_cli.cmds.util import DASHES
 from agentc_cli.cmds.util import KIND_COLORS
 from agentc_cli.cmds.util import get_catalog
+from agentc_cli.cmds.util import logging_command
 from agentc_core.catalog import CatalogBase
 from agentc_core.config import Config
 
+logger = logging.getLogger(__name__)
 
+
+@logging_command(parent_logger=logger)
 def cmd_ls(
     cfg: Config = None,
     *,
@@ -33,15 +38,7 @@ def cmd_ls(
         click.secho(DASHES, fg=KIND_COLORS[k])
         click.secho(k.upper(), bold=True, fg=KIND_COLORS[k])
         click.secho(DASHES, fg=KIND_COLORS[k])
-        catalog: CatalogBase = get_catalog(
-            catalog_path=cfg.CatalogPath(),
-            bucket=cfg.bucket,
-            cluster=cfg.Cluster() if with_db else None,
-            force=force,
-            include_dirty=include_dirty,
-            kind=k,
-        )
-
+        catalog: CatalogBase = get_catalog(cfg, force=force, include_dirty=include_dirty, kind=k)
         catalog_items = list(catalog)
         num = 1
         for catalog_item in catalog_items:

libs/agentc_cli/agentc_cli/cmds/publish.py

@@ -5,9 +5,11 @@
 import pathlib
 import tqdm
 import typing
+import uuid
 
 from .util import DASHES
 from .util import KIND_COLORS
+from agentc_cli.cmds.util import logging_command
 from agentc_core.analytics import Log
 from agentc_core.catalog.descriptor import CatalogDescriptor
 from agentc_core.config import Config
@@ -28,6 +30,7 @@
 logger = logging.getLogger(__name__)
 
 
+@logging_command(parent_logger=logger)
 def cmd_publish(
     cfg: Config = None,
     *,
@@ -155,7 +158,7 @@ def cmd_publish(
         progress_bar = tqdm.tqdm(catalog_desc.items)
         for item in progress_bar:
             try:
-                key = item.identifier + "_" + metadata["version"]["identifier"]
+                key = uuid.uuid4().hex
                 progress_bar.set_description(item.name)
 
                 # serialise object to str
@@ -169,5 +172,5 @@ def cmd_publish(
                 cb_coll.upsert(key, item_json)
             except CouchbaseException as e:
                 click.secho(f"Couldn't insert catalog items!\n{e.message}", fg="red")
-                return e
+                raise e
         click.secho(f"{k.capitalize()} catalog items successfully uploaded to Couchbase!\n", fg="green")

libs/agentc_cli/agentc_cli/cmds/status.py

@@ -13,6 +13,7 @@
 from ..cmds.util import KIND_COLORS
 from ..cmds.util import LEVEL_COLORS
 from ..cmds.util import load_repository
+from .util import logging_command
 from agentc_core.catalog.descriptor import CatalogDescriptor
 from agentc_core.catalog.index import MetaVersion
 from agentc_core.catalog.index import index_catalog_start
@@ -33,6 +34,7 @@
 logger = logging.getLogger(__name__)
 
 
+@logging_command(logger)
 def cmd_status(
     cfg: Config = None,
     *,
@@ -269,7 +271,11 @@ def get_local_status(
                 # Start a CatalogMem on-the-fly that incorporates the dirty
                 # source file items which we'll use instead of the local catalog file.
                 errs, catalog, uninitialized_items = index_catalog_start(
-                    EmbeddingModel(embedding_model_name=catalog_desc.embedding_model),
+                    EmbeddingModel(
+                        embedding_model_name=cfg.embedding_model_name,
+                        embedding_model_auth=cfg.embedding_model_auth,
+                        embedding_model_url=cfg.embedding_model_url,
+                    ),
                     MetaVersion(
                         schema_version=catalog_desc.schema_version, library_version=catalog_desc.library_version
                     ),