refactor to make folders less dense (#109)

* refactor to make folders less dense

* update embeddings

README.md

@@ -38,7 +38,7 @@ Follow these steps to setup the Automata environment
 # Copy the env and setup files
 cp .setup.sh.example setup.sh && cp .env.example .env
 
-# Allow for execution  
+# Allow for execution
 chmod 755 setup.sh
 
 # Update the setup and env files with your local paths
@@ -103,8 +103,9 @@ Sometimes the best way to understand a complicated system is to start by underst
 import logging
 from automata.config.openai_agent import AutomataOpenAIAgentConfigBuilder
 from automata.core.agent.providers import OpenAIAutomataAgent
-from automata.core.agent.tool.tool_utils import AgentToolFactory, dependency_factory
-from automata.core.coding.py.module_loader import py_module_loader
+from automata.core.tools.tool_utils import AgentToolFactory
+from automata.core.singletons.dependency_factory import dependency_factory
+from automata.core.singletons.module_loader import py_module_loader
 
 logger = logging.getLogger(__name__)
 
@@ -142,9 +143,10 @@ Examples of these classes are:
 `SymbolDocEmbedding` a class used for storing embeddings related to the documentation of a symbol.
 
 Code example for creating an instance of 'SymbolCodeEmbedding':
+
 ```python
 import numpy as np
-from automata.core.base.symbol_embedding import SymbolCodeEmbedding
+from automata.core.symbol_embedding.base import SymbolCodeEmbedding
 from automata.core.symbol.parser import parse_symbol
 
 symbol_str = 'scip-python python automata 75482692a6fe30c72db516201a6f47d9fb4af065 `automata.core.agent.agent_enums`/ActionIndicator#'
@@ -156,8 +158,9 @@ embedding = SymbolCodeEmbedding(symbol=symbol, source_code=source_code, vector=v
 ```
 
 Code example for creating an instance of 'SymbolDocEmbedding':
+
 ```python
-from automata.core.base.symbol_embedding import SymbolDocEmbedding
+from automata.core.symbol_embedding.base import SymbolDocEmbedding
 from automata.core.symbol.parser import parse_symbol
 import numpy as np
 
@@ -185,7 +188,6 @@ discussion, and please direct specific questions.**
 The Automata project strives to abide by generally accepted best practices in
 open-source software development.
 
-
 ## Future
 
 The ultimate goal of the Automata project is to achieve a level of proficiency where it can independently design, write, test, and refine complex software systems. This includes the ability to understand and navigate large codebases, reason about software architecture, optimize performance, and even invent new algorithms or data structures when necessary.

automata/cli/scripts/run_agent.py

@@ -5,9 +5,10 @@
 from automata.config.base import AgentConfigName
 from automata.config.openai_agent import AutomataOpenAIAgentConfigBuilder
 from automata.core.agent.providers import OpenAIAutomataAgent
-from automata.core.agent.tool.tool_utils import AgentToolFactory, dependency_factory
-from automata.core.base.github_manager import GitHubManager
-from automata.core.coding.py.module_loader import py_module_loader
+from automata.core.github_management.client import GitHubClient
+from automata.core.singletons.dependency_factory import dependency_factory
+from automata.core.singletons.module_loader import py_module_loader
+from automata.core.tools.factory import AgentToolFactory
 
 logger = logging.getLogger(__name__)
 
@@ -17,7 +18,7 @@
 # Solve the GitHub issues by writing the relevant code via the PyWriter tool. The issues begin now:"""
 
 
-def process_issues(issue_numbers: List[int], github_manager: GitHubManager) -> List[str]:
+def process_issues(issue_numbers: List[int], github_manager: GitHubClient) -> List[str]:
     """
     Process the issues and create tasks for each of them.
 
@@ -41,7 +42,7 @@ def process_issues(issue_numbers: List[int], github_manager: GitHubManager) -> L
 
 def main(*args, **kwargs):
     py_module_loader.initialize()
-    github_manager = GitHubManager(access_token=GITHUB_API_KEY, remote_name=REPOSITORY_NAME)
+    github_manager = GitHubClient(access_token=GITHUB_API_KEY, remote_name=REPOSITORY_NAME)
 
     # Pre-process issues if they are passsed
     issue_numbers = kwargs.get("fetch_issues", "")

automata/cli/scripts/run_code_embedding.py

@@ -4,12 +4,12 @@
 from tqdm import tqdm
 
 from automata.config.base import ConfigCategory
-from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase
-from automata.core.coding.py.module_loader import py_module_loader
 from automata.core.llm.providers.openai import OpenAIEmbeddingProvider
 from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler
+from automata.core.singletons.module_loader import py_module_loader
 from automata.core.symbol.graph import SymbolGraph
 from automata.core.symbol.symbol_utils import get_rankable_symbols
+from automata.core.symbol_embedding.base import JSONSymbolEmbeddingVectorDatabase
 from automata.core.symbol_embedding.builders import SymbolCodeEmbeddingBuilder
 from automata.core.utils import get_config_fpath
 

automata/cli/scripts/run_doc_embedding_l2.py

@@ -4,23 +4,23 @@
 from tqdm import tqdm
 
 from automata.config.base import ConfigCategory
-from automata.core.base.symbol import SymbolDescriptor
-from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase
-from automata.core.coding.py.module_loader import py_module_loader
-from automata.core.context.py.retriever import (
-    PyContextRetriever,
-    PyContextRetrieverConfig,
-)
+from automata.core.experimental.search.rank import SymbolRankConfig
+from automata.core.experimental.search.symbol_search import SymbolSearch
 from automata.core.llm.providers.openai import (
     OpenAIChatCompletionProvider,
     OpenAIEmbeddingProvider,
 )
 from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler
 from automata.core.memory_store.symbol_doc_embedding import SymbolDocEmbeddingHandler
+from automata.core.retrievers.py.context import (
+    PyContextRetriever,
+    PyContextRetrieverConfig,
+)
+from automata.core.singletons.module_loader import py_module_loader
+from automata.core.symbol.base import SymbolDescriptor
 from automata.core.symbol.graph import SymbolGraph
-from automata.core.symbol.search.rank import SymbolRankConfig
-from automata.core.symbol.search.symbol_search import SymbolSearch
 from automata.core.symbol.symbol_utils import get_rankable_symbols
+from automata.core.symbol_embedding.base import JSONSymbolEmbeddingVectorDatabase
 from automata.core.symbol_embedding.builders import (
     SymbolCodeEmbeddingBuilder,
     SymbolDocEmbeddingBuilder,

automata/cli/scripts/run_doc_embedding_l3.py

@@ -4,23 +4,23 @@
 from tqdm import tqdm
 
 from automata.config.base import ConfigCategory
-from automata.core.base.symbol import SymbolDescriptor
-from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase
-from automata.core.coding.py.module_loader import py_module_loader
-from automata.core.context.py.retriever import (
-    PyContextRetriever,
-    PyContextRetrieverConfig,
-)
+from automata.core.experimental.search.rank import SymbolRankConfig
+from automata.core.experimental.search.symbol_search import SymbolSearch
 from automata.core.llm.providers.openai import (
     OpenAIChatCompletionProvider,
     OpenAIEmbeddingProvider,
 )
 from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler
 from automata.core.memory_store.symbol_doc_embedding import SymbolDocEmbeddingHandler
+from automata.core.retrievers.py.context import (
+    PyContextRetriever,
+    PyContextRetrieverConfig,
+)
+from automata.core.singletons.module_loader import py_module_loader
+from automata.core.symbol.base import SymbolDescriptor
 from automata.core.symbol.graph import SymbolGraph
-from automata.core.symbol.search.rank import SymbolRankConfig
-from automata.core.symbol.search.symbol_search import SymbolSearch
 from automata.core.symbol.symbol_utils import get_rankable_symbols
+from automata.core.symbol_embedding.base import JSONSymbolEmbeddingVectorDatabase
 from automata.core.symbol_embedding.builders import (
     SymbolCodeEmbeddingBuilder,
     SymbolDocEmbeddingBuilder,

automata/cli/scripts/run_doc_post_process.py

@@ -2,8 +2,8 @@
 import os
 
 from automata.config.base import ConfigCategory
-from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase
-from automata.core.coding.py.writer import PyDocWriter
+from automata.core.code_handling.py.writer import PyDocWriter
+from automata.core.symbol_embedding.base import JSONSymbolEmbeddingVectorDatabase
 from automata.core.utils import get_config_fpath, get_root_py_fpath
 
 logger = logging.getLogger(__name__)

automata/config/base.py

@@ -6,7 +6,7 @@
 import yaml
 from pydantic import BaseModel, PrivateAttr
 
-from automata.core.base.tool import Tool
+from automata.core.tools.base import Tool
 from automata.core.utils import convert_kebab_to_snake
 
 

automata/config/openai_agent.py

@@ -10,8 +10,8 @@
     InstructionConfigVersion,
     LLMProvider,
 )
-from automata.core.agent.tool.tool_utils import dependency_factory
-from automata.core.symbol.search.rank import SymbolRank
+from automata.core.experimental.search.rank import SymbolRank
+from automata.core.singletons.dependency_factory import dependency_factory
 
 
 class AutomataOpenAIAgentConfig(AgentConfig):

automata/core/base/agent.py → automata/core/agent/agent.py

@@ -6,11 +6,11 @@
 from pydantic import BaseModel
 
 from automata.config.base import AgentConfigName, LLMProvider
-from automata.core.base.tool import Tool
 from automata.core.llm.foundation import (
     LLMConversationDatabaseProvider,
     LLMIterationResult,
 )
+from automata.core.tools.base import Tool
 
 logger = logging.getLogger(__name__)
 

automata/core/agent/instances.py

@@ -4,7 +4,7 @@
 
 from automata.config.base import AgentConfigName
 from automata.config.openai_agent import AutomataOpenAIAgentConfigBuilder
-from automata.core.base.agent import AgentInstance
+from automata.core.agent.agent import AgentInstance
 
 if TYPE_CHECKING:
     from automata.core.agent.providers import OpenAIAutomataAgent

automata/core/agent/providers.py

@@ -4,14 +4,14 @@
 
 from automata.config.base import ConfigCategory
 from automata.config.openai_agent import AutomataOpenAIAgentConfig
+from automata.core.agent.agent import Agent, AgentToolkit
 from automata.core.agent.error import (
     AgentDatabaseError,
     AgentGeneralError,
     AgentMaxIterError,
     AgentResultError,
     AgentStopIteration,
 )
-from automata.core.base.agent import Agent, AgentToolkit
 from automata.core.llm.foundation import (
     LLMChatMessage,
     LLMConversationDatabaseProvider,