From 820e94bea1bb8f729289a535ea347b744eb214ce Mon Sep 17 00:00:00 2001 From: qinhui <> Date: Tue, 21 Jan 2025 19:49:38 +0800 Subject: [PATCH 1/2] Add home assistant tool --- agents/examples/demo/manifest.json | 5 + agents/examples/demo/property.json | 315 +++++++++++++++ .../home_assistant_tool_python/README.md | 29 ++ .../home_assistant_tool_python/__init__.py | 6 + .../home_assistant_tool_python/addon.py | 18 + .../home_assistant_tool_python/client.py | 110 +++++ .../device_controller.py | 157 ++++++++ .../home_assistant_tool_python/extension.py | 377 ++++++++++++++++++ .../home_assistant_tool_python/manifest.json | 83 ++++ .../home_assistant_tool_python/property.json | 1 + .../tests/conftest.py | 36 ++ .../tests/test_basic.py | 54 +++ 12 files changed, 1191 insertions(+) create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/README.md create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/__init__.py create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/addon.py create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/client.py create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/device_controller.py create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/extension.py create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/manifest.json create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/property.json create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/tests/conftest.py create mode 100644 agents/ten_packages/extension/home_assistant_tool_python/tests/test_basic.py diff --git a/agents/examples/demo/manifest.json b/agents/examples/demo/manifest.json index 8491519c3..67bddff19 100644 --- a/agents/examples/demo/manifest.json +++ b/agents/examples/demo/manifest.json @@ -63,6 +63,11 @@ "name": "weatherapi_tool_python", "version": "=0.1.0" }, + { + "type": "extension", + "name": "home_assistant_tool_python", + "version": "=0.1.0" + }, { "type": "extension", "name": "interrupt_detector_python", diff --git a/agents/examples/demo/property.json b/agents/examples/demo/property.json index 6d837b4d2..198c95677 100644 --- a/agents/examples/demo/property.json +++ b/agents/examples/demo/property.json @@ -1,6 +1,321 @@ { "_ten": { "predefined_graphs": [ + { + "name": "va_home_assistant", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "name": "agora_rtc", + "addon": "agora_rtc", + "extension_group": "default", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "token": "", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": true, + "agora_asr_vendor_name": "microsoft", + "agora_asr_language": "en-US", + "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}", + "agora_asr_session_control_file_path": "session_control.conf", + "subscribe_video_pix_fmt": 4, + "subscribe_video": true + } + }, + { + "type": "extension", + "name": "llm", + "addon": "openai_chatgpt_python", + "extension_group": "chatgpt", + "property": { + "api_key": "${env:OPENAI_API_KEY}", + "base_url": "", + "frequency_penalty": 0.9, + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10, + "max_tokens": 512, + "model": "${env:OPENAI_MODEL}", + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL}" + } + }, + { + "type": "extension", + "name": "tts", + "addon": "azure_tts", + "extension_group": "tts", + "property": { + "azure_subscription_key": "${env:AZURE_TTS_KEY}", + "azure_subscription_region": "${env:AZURE_TTS_REGION}", + "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + } + }, + { + "type": "extension", + "name": "interrupt_detector", + "addon": "interrupt_detector_python", + "extension_group": "default", + "property": {} + }, + { + "type": "extension", + "name": "message_collector", + "addon": "message_collector", + "extension_group": "transcriber", + "property": {} + }, + { + "type": "extension", + "name": "weatherapi_tool_python", + "addon": "weatherapi_tool_python", + "extension_group": "default", + "property": { + "api_key": "${env:WEATHERAPI_API_KEY|}" + } + }, + { + "type": "extension", + "name": "vision_tool_python", + "addon": "vision_tool_python", + "extension_group": "default", + "property": {} + }, + { + "type": "extension", + "name": "bingsearch_tool_python", + "addon": "bingsearch_tool_python", + "extension_group": "default", + "property": { + "api_key": "${env:BING_API_KEY|}" + } + }, + { + "type": "extension", + "name": "home_assistant_tool_python", + "addon": "home_assistant_tool_python", + "extension_group": "default", + "property": { + "api_key": "${env:HOME_ASSISTANT_API_KEY|}", + "base_url": "${env:HOME_ASSISTANT_BASE_URL|}" + } + } + ], + "connections": [ + { + "extension": "agora_rtc", + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension": "llm" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension": "llm" + } + ] + }, + { + "name": "on_connection_failure", + "dest": [ + { + "extension": "llm" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension": "interrupt_detector" + }, + { + "extension": "message_collector" + } + ] + } + ], + "video_frame": [ + { + "name": "video_frame", + "dest": [ + { + "extension": "vision_tool_python" + } + ] + } + ] + }, + { + "extension": "llm", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension": "tts" + } + ] + }, + { + "name": "tool_call", + "dest": [ + { + "extension": "weatherapi_tool_python" + }, + { + "extension": "vision_tool_python" + }, + { + "extension": "bingsearch_tool_python" + }, + { + "extension": "home_assistant_tool_python" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension": "tts" + }, + { + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension": "message_collector", + "data": [ + { + "name": "data", + "dest": [ + { + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension": "tts", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension": "agora_rtc" + } + ] + } + ], + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "agora_rtc" + } + ] + } + ] + }, + { + "extension": "interrupt_detector", + "cmd": [ + { + "name": "flush", + "dest": [ + { + "extension": "llm" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension": "llm" + } + ] + } + ] + }, + { + "extension": "weatherapi_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension": "llm" + } + ] + } + ] + }, + { + "extension": "vision_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension": "llm" + } + ] + } + ] + }, + { + "extension": "bingsearch_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension": "llm" + } + ] + } + ] + }, + { + "extension": "home_assistant_tool_python", + "cmd": [ + { + "name": "tool_register", + "dest": [ + { + "extension": "llm" + } + ] + } + ] + } + ] + }, { "name": "va_openai_azure", "auto_start": true, diff --git a/agents/ten_packages/extension/home_assistant_tool_python/README.md b/agents/ten_packages/extension/home_assistant_tool_python/README.md new file mode 100644 index 000000000..5880b33b3 --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/README.md @@ -0,0 +1,29 @@ +# home_assistant_tool_python + + + +## Features + + + +- xxx feature + +## API + +Refer to `api` definition in [manifest.json] and default values in [property.json](property.json). + + + +## Development + +### Build + + + +### Unit test + + + +## Misc + + diff --git a/agents/ten_packages/extension/home_assistant_tool_python/__init__.py b/agents/ten_packages/extension/home_assistant_tool_python/__init__.py new file mode 100644 index 000000000..72593ab22 --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/__init__.py @@ -0,0 +1,6 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from . import addon diff --git a/agents/ten_packages/extension/home_assistant_tool_python/addon.py b/agents/ten_packages/extension/home_assistant_tool_python/addon.py new file mode 100644 index 000000000..9c655ecda --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/addon.py @@ -0,0 +1,18 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import ( + Addon, + register_addon_as_extension, + TenEnv, +) + + +@register_addon_as_extension("home_assistant_tool_python") +class HomeAssistantExtensionAddon(Addon): + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + from .extension import HomeAssistantExtension + ten_env.log_info("on_create_instance") + ten_env.on_create_instance_done(HomeAssistantExtension(name), context) diff --git a/agents/ten_packages/extension/home_assistant_tool_python/client.py b/agents/ten_packages/extension/home_assistant_tool_python/client.py new file mode 100644 index 000000000..67f1cf121 --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/client.py @@ -0,0 +1,110 @@ +import aiohttp +from typing import Optional, Dict, Any + +class HomeAssistantAPI: + """Home Assistant REST API client""" + + def __init__(self, host: str, token: str, verify_ssl: bool = True, timeout: int = 10): + """Initialize the client + + Args: + host: Home Assistant address (e.g., http://192.168.1.100:8123) + token: Long-lived access token + verify_ssl: Whether to verify SSL certificates + timeout: Timeout time (seconds) + """ + self.host = host.rstrip('/') + self.headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + self.verify_ssl = verify_ssl + self.timeout = timeout + self._api_url = f"{self.host}/api" + self.session: Optional[aiohttp.ClientSession] = None + + async def __aenter__(self): + """Async context manager entry""" + self.session = aiohttp.ClientSession(headers=self.headers) + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit""" + if self.session: + await self.session.close() + + async def _request(self, method: str, endpoint: str, data: Dict = None) -> Any: + """Send API request""" + if not self.session: + self.session = aiohttp.ClientSession(headers=self.headers) + + url = f"{self._api_url}/{endpoint.lstrip('/')}" + async with self.session.request(method, url, json=data, ssl=self.verify_ssl) as response: + if response.status == 401: + raise Exception("Authentication failed") + if response.status not in [200, 201]: + raise Exception(f"API request failed: {response.status}") + return await response.json() + + # API methods + async def get_config(self): + """Get Home Assistant configuration""" + return await self._request("GET", "config") + + async def get_events(self): + """Get available event list""" + return await self._request("GET", "events") + + async def get_services(self): + """Get available service list""" + return await self._request("GET", "services") + + async def get_states(self): + """Get all entity states""" + return await self._request("GET", "states") + + async def get_state(self, entity_id: str): + """Get specific entity state""" + return await self._request("GET", f"states/{entity_id}") + + async def set_state(self, entity_id: str, state: str, attributes: Dict = None): + """Set entity state""" + data = {"state": state} + if attributes: + data["attributes"] = attributes + return await self._request("POST", f"states/{entity_id}", data) + + async def call_service(self, domain: str, service: str, service_data: Dict = None): + """Call service""" + return await self._request("POST", f"services/{domain}/{service}", service_data) + + # Device control methods + async def control_device(self, domain: str, service: str, entity_id: str, **service_data): + """Control device""" + data = {"entity_id": entity_id, **service_data} + return await self.call_service(domain, service, data) + + async def turn_on(self, entity_id: str, **kwargs): + """Turn on device""" + domain = entity_id.split('.')[0] + return await self.control_device(domain, "turn_on", entity_id, **kwargs) + + async def turn_off(self, entity_id: str, **kwargs): + """Turn off device""" + domain = entity_id.split('.')[0] + return await self.control_device(domain, "turn_off", entity_id, **kwargs) + + # Xiaomi device specific methods + async def set_xiaomi_light(self, entity_id: str, brightness: Optional[int] = None, + color_temp: Optional[int] = None): + """Control Xiaomi light""" + data = {} + if brightness is not None: + data["brightness"] = brightness + if color_temp is not None: + data["color_temp"] = color_temp + return await self.turn_on(entity_id, **data) + + async def set_xiaomi_fan_speed(self, entity_id: str, speed: str): + """Set Xiaomi fan speed""" + return await self.control_device("fan", "set_speed", entity_id, speed=speed) \ No newline at end of file diff --git a/agents/ten_packages/extension/home_assistant_tool_python/device_controller.py b/agents/ten_packages/extension/home_assistant_tool_python/device_controller.py new file mode 100644 index 000000000..88a9750b0 --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/device_controller.py @@ -0,0 +1,157 @@ +from typing import List, Dict, Any, Optional +from .client import HomeAssistantAPI + +class DeviceController: + def __init__(self, host: str, token: str): + self.ha = HomeAssistantAPI(host, token) + + async def __aenter__(self): + await self.ha.__aenter__() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.ha.__aexit__(exc_type, exc_val, exc_tb) + + async def get_all_devices(self) -> List[Dict[str, Any]]: + """Get all controllable devices + + Returns: + List[Dict]: Device list, each device includes ID, type, state, and supported operations + """ + devices = [] + states = await self.ha.get_states() + services = await self.ha.get_services() + + # Controllable device types + controllable_domains = { + 'light', # Light + 'switch', # Switch + 'climate', # Air Conditioner/Heater + 'fan', # Fan + 'cover', # Curtain + 'media_player', # Media Player + 'vacuum', # Vacuum Cleaner + 'water_heater', # Water Heater + 'camera', # Camera + } + + for state in states: + entity_id = state['entity_id'] + domain = entity_id.split('.')[0] + + # Only include controllable devices + if domain not in controllable_domains: + continue + + # Get the services supported by this domain + domain_services = [] + for service in services: + if service['domain'] == domain: + domain_services = list(service.get('services', {}).keys()) + break + + # Only include devices with available services + if not domain_services: + continue + + device = { + 'entity_id': entity_id, + 'name': state['attributes'].get('friendly_name', entity_id), + 'domain': domain, + 'state': state['state'], + 'attributes': state['attributes'], + 'supported_services': domain_services + } + devices.append(device) + + return devices + + async def control_device(self, + entity_id: str, + action: str, + **params) -> Dict[str, Any]: + """General method for controlling devices + + Args: + entity_id: Device ID (e.g., light.bed_light) + action: Action name (e.g., turn_on, turn_off, set_brightness) + **params: Action parameters + + Returns: + Dict: Operation result + """ + domain = entity_id.split('.')[0] + + # Special action handling + if action == 'set_brightness' and domain == 'light': + result = await self.ha.turn_on(entity_id, brightness=params.get('brightness', 255)) + elif action == 'set_temperature' and domain == 'climate': + result = await self.ha.control_device(domain, 'set_temperature', + entity_id, temperature=params.get('temperature')) + else: + result = await self.ha.control_device(domain, action, entity_id, **params) + + # Get the updated state + new_state = await self.ha.get_state(entity_id) + + return { + 'success': True, + 'entity_id': entity_id, + 'action': action, + 'params': params, + 'result': result, + 'new_state': new_state + } + + async def get_device_info(self, entity_id: str) -> Dict[str, Any]: + """Get detailed information of a single device + + Args: + entity_id: Device ID + + Returns: + Dict: Device detailed information + """ + state = await self.ha.get_state(entity_id) + domain = entity_id.split('.')[0] + + services = await self.ha.get_services() + domain_services = [] + for service in services: + if service['domain'] == domain: + domain_services = list(service.get('services', {}).keys()) + break + + return { + 'entity_id': entity_id, + 'name': state['attributes'].get('friendly_name', entity_id), + 'domain': domain, + 'state': state['state'], + 'attributes': state['attributes'], + 'supported_services': domain_services + } + + async def get_devices_by_type(self) -> Dict[str, List[Dict[str, Any]]]: + """Get all controllable devices by type + + Returns: + Dict[str, List[Dict]]: Device list categorized by device type + { + 'light': [device list], + 'switch': [device list], + ... + } + """ + devices = await self.get_all_devices() + + categorized_devices = {} + for device in devices: + domain = device['domain'] + if domain not in categorized_devices: + categorized_devices[domain] = { + 'name': domain, + 'devices': [] + } + categorized_devices[domain]['devices'].append(device) + + return categorized_devices \ No newline at end of file diff --git a/agents/ten_packages/extension/home_assistant_tool_python/extension.py b/agents/ten_packages/extension/home_assistant_tool_python/extension.py new file mode 100644 index 000000000..657ecf269 --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/extension.py @@ -0,0 +1,377 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import json +from typing import Any, Dict, List +from dataclasses import dataclass +from ten_ai_base.config import BaseConfig +from ten_ai_base import AsyncLLMToolBaseExtension +from ten_ai_base.types import LLMToolMetadata, LLMToolMetadataParameter, LLMToolResult, LLMToolResultLLMResult +from .device_controller import DeviceController + +from ten import ( + AsyncTenEnv, + Cmd, +) + +LIGHT_TOOL_NAME = "light_control" +CAMERA_TOOL_NAME = "camera_control" +CLIMATE_TOOL_NAME = "climate_control" +COVER_TOOL_NAME = "cover_control" +MEDIA_PLAYER_TOOL_NAME = "media_player_control" +FAN_TOOL_NAME = "fan_control" + +@dataclass +class HomeAssistantConfig(BaseConfig): + base_url: str = "" + api_key: str = "" + +class HomeAssistantExtension(AsyncLLMToolBaseExtension): + def __init__(self, name: str) -> None: + super().__init__(name) + self.session = None + self.config: HomeAssistantConfig = None + self.device_controller: DeviceController = None + self.categorized_devices: Dict[str, List[Dict[str, Any]]] = {} + self.ten_env: AsyncTenEnv = None + async def on_init(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_init") + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_start") + self.ten_env = ten_env + self.config = await HomeAssistantConfig.create_async(ten_env=ten_env) + ten_env.log_info(f"config: {self.config}") + if self.config.api_key and self.config.base_url: + self.device_controller = DeviceController(self.config.base_url, self.config.api_key) + self.categorized_devices = await self._get_all_devices() + ten_env.log_info("get all devices finished") + + await super().on_start(ten_env) + + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_stop") + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + cmd_name = cmd.get_name() + ten_env.log_debug("on_cmd name {}".format(cmd_name)) + + await super().on_cmd(ten_env, cmd) + + def get_tool_metadata(self, ten_env: AsyncTenEnv) -> list[LLMToolMetadata]: + if not self.categorized_devices: + ten_env.log_info("categorized_devices is empty") + return [] + + metadata_list = [] + for domain, category in self.categorized_devices.items(): + if domain == "light": + metadata = self.light_metadata(domain, category['devices'], ten_env) + metadata_list.append(metadata) + ten_env.log_info(f"will register tool: {metadata.name}") + elif domain == "camera": + metadata = self.camera_metadata(domain, category['devices'], ten_env) + metadata_list.append(metadata) + ten_env.log_info(f"will register tool: {metadata.name}") + elif domain == "climate": + metadata = self.climate_metadata(domain, category['devices'], ten_env) + metadata_list.append(metadata) + ten_env.log_info(f"will register tool: {metadata.name}") + elif domain == "cover": + metadata = self.cover_metadata(domain, category['devices'], ten_env) + metadata_list.append(metadata) + ten_env.log_info(f"will register tool: {metadata.name}") + elif domain == "media_player": + metadata = self.media_player_metadata(domain, category['devices'], ten_env) + metadata_list.append(metadata) + ten_env.log_info(f"will register tool: {metadata.name}") + elif domain == "fan": + metadata = self.fan_metadata(domain, category['devices'], ten_env) + metadata_list.append(metadata) + ten_env.log_info(f"will register tool: {metadata.name}") + + return metadata_list + + def tool_description(self, device_type: str, action: str, devices: List[Dict[str, Any]], ten_env: AsyncTenEnv) -> str: + system_prompt = f"Control the {device_type} in the user's home based on voice commands." + user_prompt = action + device_info = f"Devices:\n" + + unique_services = set() + for device in devices: + unique_services.update(device['supported_services']) + device_info += f"{device['name']} ({device['entity_id']})\n" + + device_info += f"Supported services: {list(unique_services)}\n" + user_prompt += device_info + + tool_description = f"{system_prompt}\n{user_prompt}" + return tool_description + + def fan_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_env: AsyncTenEnv) -> LLMToolMetadata: + action = f""" + 1. Turn on the {device_type} + 2. Turn off the {device_type} + """ + tool_description = self.tool_description(device_type, action, devices, ten_env) + ten_env.log_info(f"fan_tool_name: {FAN_TOOL_NAME}, fan_tool_description: {tool_description}") + + return LLMToolMetadata( + name=FAN_TOOL_NAME, + description=tool_description, + parameters=[ + LLMToolMetadataParameter( + name="entity_id", + type="string", + description="The entity_id of the device to control", + required=True, + ), + LLMToolMetadataParameter( + name="action", + type="string", + description="The action to perform on the device, such as turn on, turn off etc.", + required=True, + ), + LLMToolMetadataParameter( + name="percentage", + type="string", + description="The speed to set, in percentage", + required=False, + ), + ], + ) + + def cover_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_env: AsyncTenEnv) -> LLMToolMetadata: + action = f""" + 1. Open the {device_type} + 2. Close the {device_type} + """ + tool_description = self.tool_description(device_type, action, devices, ten_env) + ten_env.log_info(f"cover_tool_name: {COVER_TOOL_NAME}, cover_tool_description: {tool_description}") + + return LLMToolMetadata( + name=COVER_TOOL_NAME, + description=tool_description, + parameters=[ + LLMToolMetadataParameter( + name="entity_id", + type="string", + description="The entity_id of the device to control", + required=True, + ), + LLMToolMetadataParameter( + name="action", + type="string", + description="The action to perform on the device, such as turn on, turn off etc.", + required=True, + ), + ], + ) + + def media_player_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_env: AsyncTenEnv) -> LLMToolMetadata: + action = f""" + 1. Play the {device_type} + 2. Pause the {device_type} + 3. Stop the {device_type} + """ + tool_description = self.tool_description(device_type, action, devices, ten_env) + ten_env.log_info(f"media_player_tool_name: {MEDIA_PLAYER_TOOL_NAME}, media_player_tool_description: {tool_description}") + + return LLMToolMetadata( + name=MEDIA_PLAYER_TOOL_NAME, + description=tool_description, + parameters=[ + LLMToolMetadataParameter( + name="entity_id", + type="string", + description="The entity_id of the device to control", + required=True, + ), + LLMToolMetadataParameter( + name="action", + type="string", + description="The action to perform on the device, such as play, pause, stop etc.", + required=True, + ), + LLMToolMetadataParameter( + name="volume", + type="string", + description="The volume to set, in percentage", + required=False, + ), + ], + ) + + def climate_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_env: AsyncTenEnv) -> LLMToolMetadata: + action = f""" + 1. Turn on the {device_type} + 2. Turn off the {device_type} + """ + tool_description = self.tool_description(device_type, action, devices, ten_env) + ten_env.log_info(f"climate_tool_name: {CLIMATE_TOOL_NAME}, climate_tool_description: {tool_description}") + + return LLMToolMetadata( + name=CLIMATE_TOOL_NAME, + description=tool_description, + parameters=[ + LLMToolMetadataParameter( + name="entity_id", + type="string", + description="The entity_id of the device to control", + required=True, + ), + LLMToolMetadataParameter( + name="action", + type="string", + description="The action to perform on the device, such as turn on, turn off etc.", + required=True, + ), + LLMToolMetadataParameter( + name="temperature", + type="number", + description="The temperature to set, in degrees Celsius", + required=False, + ), + ], + ) + + def camera_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_env: AsyncTenEnv) -> LLMToolMetadata: + action = f""" + 1. Turn on the {device_type} + 2. Turn off the {device_type} + """ + tool_description = self.tool_description(device_type, action, devices, ten_env) + ten_env.log_info(f"camera_tool_name: {CAMERA_TOOL_NAME}, camera_tool_description: {tool_description}") + + return LLMToolMetadata( + name=CAMERA_TOOL_NAME, + description=tool_description, + parameters=[ + LLMToolMetadataParameter( + name="entity_id", + type="string", + description="The entity_id of the device to control", + required=True, + ), + LLMToolMetadataParameter( + name="action", + type="string", + description="The action to perform on the device, such as turn on, turn off etc.", + required=True, + ), + ], + ) + + def light_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_env: AsyncTenEnv) -> LLMToolMetadata: + action = f""" + 1. Turn on the {device_type} + 2. Turn off the {device_type} + """ + tool_description = self.tool_description(device_type, action, devices, ten_env) + + ten_env.log_info(f"light_tool_name: {LIGHT_TOOL_NAME}, light_tool_description: {tool_description}") + return LLMToolMetadata( + name=LIGHT_TOOL_NAME, + description=tool_description, + parameters=[ + LLMToolMetadataParameter( + name="entity_id", + type="string", + description="The entity_id of the device to control", + required=True, + ), + LLMToolMetadataParameter( + name="action", + type="string", + description="The action to perform on the device, such as turn on, turn off etc.", + required=True, + ), + ], + ) + + async def run_tool(self, ten_env: AsyncTenEnv, name: str, args: dict) -> LLMToolResult | None: + ten_env.log_info(f"run_tool name: {name}, args: {args}") + entity_id = args.get("entity_id") + action = args.get("action") + if name == LIGHT_TOOL_NAME: + result = await self._control_light(entity_id, action) + return LLMToolResultLLMResult( + type="llmresult", + content=json.dumps(result), + ) + elif name == CAMERA_TOOL_NAME: + result = await self._control_camera(entity_id, action) + return LLMToolResultLLMResult( + type="llmresult", + content=json.dumps(result), + ) + elif name == CLIMATE_TOOL_NAME: + temperature = args.get("temperature") + result = await self._control_climate(entity_id, action, temperature=temperature) + return LLMToolResultLLMResult( + type="llmresult", + content=json.dumps(result), + ) + elif name == COVER_TOOL_NAME: + result = await self._control_cover(entity_id, action) + return LLMToolResultLLMResult( + type="llmresult", + content=json.dumps(result), + ) + elif name == MEDIA_PLAYER_TOOL_NAME: + volume = args.get("volume") + result = await self._control_media_player(entity_id, action, volume=volume) + return LLMToolResultLLMResult( + type="llmresult", + content=json.dumps(result), + ) + elif name == FAN_TOOL_NAME: + percentage = args.get("percentage") + result = await self._control_fan(entity_id, action, percentage=percentage) + return LLMToolResultLLMResult( + type="llmresult", + content=json.dumps(result), + ) + return None + + async def _get_all_devices(self) -> Dict[str, List[Dict[str, Any]]]: + categorized_devices = await self.device_controller.get_devices_by_type() + return categorized_devices + + async def _control_device(self, entity_id: str, action: str) -> Any: + self.ten_env.log_info(f"control_device entity_id: {entity_id}, action: {action}") + result = await self.device_controller.control_device(entity_id, action) + return result + + async def _control_cover(self, entity_id: str, action: str) -> Any: + return await self._control_device(entity_id, action) + + async def _control_light(self, entity_id: str, action: str) -> Any: + return await self._control_device(entity_id, action) + + async def _control_camera(self, entity_id: str, action: str) -> Any: + return await self._control_device(entity_id, action) + + async def _control_media_player(self, entity_id: str, action: str, **kwargs) -> Any: + self.ten_env.log_info(f"control_media_player entity_id: {entity_id}, action: {action}, kwargs: {kwargs}") + if kwargs.get("volume") is None: + return await self._control_device(entity_id, action) + + return await self.device_controller.control_device(entity_id, action, **kwargs) + + async def _control_climate(self, entity_id: str, action: str, **kwargs) -> Any: + self.ten_env.log_info(f"control_climate entity_id: {entity_id}, action: {action}, kwargs: {kwargs}") + if kwargs.get("temperature") is None: + return await self._control_device(entity_id, action) + + return await self.device_controller.control_device(entity_id, action, **kwargs) + + async def _control_fan(self, entity_id: str, action: str, **kwargs) -> Any: + self.ten_env.log_info(f"control_fan entity_id: {entity_id}, action: {action}, kwargs: {kwargs}") + if kwargs.get("percentage") is None: + return await self._control_device(entity_id, action) + + return await self.device_controller.control_device(entity_id, action, **kwargs) diff --git a/agents/ten_packages/extension/home_assistant_tool_python/manifest.json b/agents/ten_packages/extension/home_assistant_tool_python/manifest.json new file mode 100644 index 000000000..d06ea5e19 --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/manifest.json @@ -0,0 +1,83 @@ +{ + "type": "extension", + "name": "home_assistant_tool_python", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.6" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "BUILD.gn", + "**.tent", + "**.py", + "README.md", + "tests/**" + ] + }, + "api": { + "property": { + "api_key": { + "type": "string" + } + }, + "cmd_out": [ + { + "name": "tool_register", + "property": { + "tool": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "properties": {} + } + } + }, + "required": [ + "name", + "description", + "parameters" + ] + } + }, + "result": { + "property": { + "response": { + "type": "string" + } + } + } + } + ], + "cmd_in": [ + { + "name": "tool_call", + "property": { + "name": { + "type": "string" + }, + "args": { + "type": "string" + } + }, + "required": [ + "name" + ] + } + ] + } +} \ No newline at end of file diff --git a/agents/ten_packages/extension/home_assistant_tool_python/property.json b/agents/ten_packages/extension/home_assistant_tool_python/property.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/agents/ten_packages/extension/home_assistant_tool_python/tests/conftest.py b/agents/ten_packages/extension/home_assistant_tool_python/tests/conftest.py new file mode 100644 index 000000000..27f004ff0 --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/tests/conftest.py @@ -0,0 +1,36 @@ +# +# Copyright © 2025 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import pytest +import sys +import os +from ten import ( + unregister_all_addons_and_cleanup, +) + + +@pytest.fixture(scope="session", autouse=True) +def global_setup_and_teardown(): + # Set the environment variable. + os.environ["TEN_DISABLE_ADDON_UNREGISTER_AFTER_APP_CLOSE"] = "true" + + # Verify the environment variable is correctly set. + if ( + "TEN_DISABLE_ADDON_UNREGISTER_AFTER_APP_CLOSE" not in os.environ + or os.environ["TEN_DISABLE_ADDON_UNREGISTER_AFTER_APP_CLOSE"] != "true" + ): + print( + "Failed to set TEN_DISABLE_ADDON_UNREGISTER_AFTER_APP_CLOSE", + file=sys.stderr, + ) + sys.exit(1) + + # Yield control to the test; after the test execution is complete, continue + # with the teardown process. + yield + + # Teardown part. + unregister_all_addons_and_cleanup() diff --git a/agents/ten_packages/extension/home_assistant_tool_python/tests/test_basic.py b/agents/ten_packages/extension/home_assistant_tool_python/tests/test_basic.py new file mode 100644 index 000000000..147fd88e4 --- /dev/null +++ b/agents/ten_packages/extension/home_assistant_tool_python/tests/test_basic.py @@ -0,0 +1,54 @@ +# +# Copyright © 2025 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +from typing import Optional +from ten import ( + ExtensionTester, + TenEnvTester, + Cmd, + CmdResult, + StatusCode, + TenError, +) + + +class ExtensionTesterBasic(ExtensionTester): + def check_hello( + self, + ten_env: TenEnvTester, + result: Optional[CmdResult], + error: Optional[TenError], + ): + if error is not None: + assert False, error.err_msg() + + assert result is not None + + statusCode = result.get_status_code() + print("receive hello_world, status:" + str(statusCode)) + + if statusCode == StatusCode.OK: + ten_env.stop_test() + + def on_start(self, ten_env: TenEnvTester) -> None: + new_cmd = Cmd.create("hello_world") + + print("send hello_world") + ten_env.send_cmd( + new_cmd, + lambda ten_env, result, error: self.check_hello( + ten_env, result, error + ), + ) + + print("tester on_start_done") + ten_env.on_start_done() + + +def test_basic(): + tester = ExtensionTesterBasic() + tester.set_test_mode_single("default_async_extension_python") + tester.run() From e870cbc2007c01de7dc192843fc054bdce4a34da Mon Sep 17 00:00:00 2001 From: qinhui <> Date: Tue, 21 Jan 2025 21:38:05 +0800 Subject: [PATCH 2/2] Light brightness control --- .../device_controller.py | 17 +++++- .../home_assistant_tool_python/extension.py | 56 ++++++++----------- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/agents/ten_packages/extension/home_assistant_tool_python/device_controller.py b/agents/ten_packages/extension/home_assistant_tool_python/device_controller.py index 88a9750b0..eb0683c92 100644 --- a/agents/ten_packages/extension/home_assistant_tool_python/device_controller.py +++ b/agents/ten_packages/extension/home_assistant_tool_python/device_controller.py @@ -154,4 +154,19 @@ async def get_devices_by_type(self) -> Dict[str, List[Dict[str, Any]]]: } categorized_devices[domain]['devices'].append(device) - return categorized_devices \ No newline at end of file + return categorized_devices + + async def set_light_brightness(self, entity_id: str, **params) -> Dict[str, Any]: + """Set the brightness of a light + + Args: + entity_id: The ID of the light entity + brightness: The brightness level (0-255) + + Returns: + Dict: Operation result + """ + + return await self.ha.turn_on(entity_id, **params) + + diff --git a/agents/ten_packages/extension/home_assistant_tool_python/extension.py b/agents/ten_packages/extension/home_assistant_tool_python/extension.py index 657ecf269..255356096 100644 --- a/agents/ten_packages/extension/home_assistant_tool_python/extension.py +++ b/agents/ten_packages/extension/home_assistant_tool_python/extension.py @@ -43,11 +43,9 @@ async def on_start(self, ten_env: AsyncTenEnv) -> None: ten_env.log_debug("on_start") self.ten_env = ten_env self.config = await HomeAssistantConfig.create_async(ten_env=ten_env) - ten_env.log_info(f"config: {self.config}") if self.config.api_key and self.config.base_url: self.device_controller = DeviceController(self.config.base_url, self.config.api_key) self.categorized_devices = await self._get_all_devices() - ten_env.log_info("get all devices finished") await super().on_start(ten_env) @@ -71,27 +69,21 @@ def get_tool_metadata(self, ten_env: AsyncTenEnv) -> list[LLMToolMetadata]: if domain == "light": metadata = self.light_metadata(domain, category['devices'], ten_env) metadata_list.append(metadata) - ten_env.log_info(f"will register tool: {metadata.name}") elif domain == "camera": metadata = self.camera_metadata(domain, category['devices'], ten_env) metadata_list.append(metadata) - ten_env.log_info(f"will register tool: {metadata.name}") elif domain == "climate": metadata = self.climate_metadata(domain, category['devices'], ten_env) metadata_list.append(metadata) - ten_env.log_info(f"will register tool: {metadata.name}") elif domain == "cover": metadata = self.cover_metadata(domain, category['devices'], ten_env) metadata_list.append(metadata) - ten_env.log_info(f"will register tool: {metadata.name}") elif domain == "media_player": metadata = self.media_player_metadata(domain, category['devices'], ten_env) metadata_list.append(metadata) - ten_env.log_info(f"will register tool: {metadata.name}") elif domain == "fan": metadata = self.fan_metadata(domain, category['devices'], ten_env) metadata_list.append(metadata) - ten_env.log_info(f"will register tool: {metadata.name}") return metadata_list @@ -117,7 +109,6 @@ def fan_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_env: 2. Turn off the {device_type} """ tool_description = self.tool_description(device_type, action, devices, ten_env) - ten_env.log_info(f"fan_tool_name: {FAN_TOOL_NAME}, fan_tool_description: {tool_description}") return LLMToolMetadata( name=FAN_TOOL_NAME, @@ -150,8 +141,7 @@ def cover_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_en 2. Close the {device_type} """ tool_description = self.tool_description(device_type, action, devices, ten_env) - ten_env.log_info(f"cover_tool_name: {COVER_TOOL_NAME}, cover_tool_description: {tool_description}") - + return LLMToolMetadata( name=COVER_TOOL_NAME, description=tool_description, @@ -176,9 +166,9 @@ def media_player_metadata(self, device_type: str, devices: List[Dict[str, Any]], 1. Play the {device_type} 2. Pause the {device_type} 3. Stop the {device_type} + 4. Volume down the {device_type} """ tool_description = self.tool_description(device_type, action, devices, ten_env) - ten_env.log_info(f"media_player_tool_name: {MEDIA_PLAYER_TOOL_NAME}, media_player_tool_description: {tool_description}") return LLMToolMetadata( name=MEDIA_PLAYER_TOOL_NAME, @@ -193,15 +183,9 @@ def media_player_metadata(self, device_type: str, devices: List[Dict[str, Any]], LLMToolMetadataParameter( name="action", type="string", - description="The action to perform on the device, such as play, pause, stop etc.", + description="The action to perform on the device, such as play, pause, stop, volume down etc.", required=True, ), - LLMToolMetadataParameter( - name="volume", - type="string", - description="The volume to set, in percentage", - required=False, - ), ], ) @@ -211,7 +195,6 @@ def climate_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_ 2. Turn off the {device_type} """ tool_description = self.tool_description(device_type, action, devices, ten_env) - ten_env.log_info(f"climate_tool_name: {CLIMATE_TOOL_NAME}, climate_tool_description: {tool_description}") return LLMToolMetadata( name=CLIMATE_TOOL_NAME, @@ -244,7 +227,6 @@ def camera_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_e 2. Turn off the {device_type} """ tool_description = self.tool_description(device_type, action, devices, ten_env) - ten_env.log_info(f"camera_tool_name: {CAMERA_TOOL_NAME}, camera_tool_description: {tool_description}") return LLMToolMetadata( name=CAMERA_TOOL_NAME, @@ -269,10 +251,10 @@ def light_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_en action = f""" 1. Turn on the {device_type} 2. Turn off the {device_type} + 3. Set brightness of the {device_type} """ tool_description = self.tool_description(device_type, action, devices, ten_env) - ten_env.log_info(f"light_tool_name: {LIGHT_TOOL_NAME}, light_tool_description: {tool_description}") return LLMToolMetadata( name=LIGHT_TOOL_NAME, description=tool_description, @@ -286,18 +268,25 @@ def light_metadata(self, device_type: str, devices: List[Dict[str, Any]], ten_en LLMToolMetadataParameter( name="action", type="string", - description="The action to perform on the device, such as turn on, turn off etc.", + description="The action to perform on the device, such as turn on, turn off, set brightness etc.", required=True, ), + LLMToolMetadataParameter( + name="brightness", + type="number", + description="The brightness level to set, from 0 to 255", + required=False, + ), ], ) async def run_tool(self, ten_env: AsyncTenEnv, name: str, args: dict) -> LLMToolResult | None: - ten_env.log_info(f"run_tool name: {name}, args: {args}") + self.ten_env.log_info(f"run tool name {name}, args {args}") entity_id = args.get("entity_id") action = args.get("action") if name == LIGHT_TOOL_NAME: - result = await self._control_light(entity_id, action) + brightness = args.get("brightness") + result = await self._control_light(entity_id, action, brightness=brightness) return LLMToolResultLLMResult( type="llmresult", content=json.dumps(result), @@ -322,8 +311,7 @@ async def run_tool(self, ten_env: AsyncTenEnv, name: str, args: dict) -> LLMTool content=json.dumps(result), ) elif name == MEDIA_PLAYER_TOOL_NAME: - volume = args.get("volume") - result = await self._control_media_player(entity_id, action, volume=volume) + result = await self._control_media_player(entity_id, action) return LLMToolResultLLMResult( type="llmresult", content=json.dumps(result), @@ -342,35 +330,35 @@ async def _get_all_devices(self) -> Dict[str, List[Dict[str, Any]]]: return categorized_devices async def _control_device(self, entity_id: str, action: str) -> Any: - self.ten_env.log_info(f"control_device entity_id: {entity_id}, action: {action}") result = await self.device_controller.control_device(entity_id, action) return result async def _control_cover(self, entity_id: str, action: str) -> Any: return await self._control_device(entity_id, action) - async def _control_light(self, entity_id: str, action: str) -> Any: - return await self._control_device(entity_id, action) + async def _control_light(self, entity_id: str, action: str, **kwargs) -> Any: + if kwargs.get("brightness") is None: + return await self._control_device(entity_id, action) + + return await self.device_controller.set_light_brightness(entity_id, **kwargs) async def _control_camera(self, entity_id: str, action: str) -> Any: return await self._control_device(entity_id, action) async def _control_media_player(self, entity_id: str, action: str, **kwargs) -> Any: - self.ten_env.log_info(f"control_media_player entity_id: {entity_id}, action: {action}, kwargs: {kwargs}") if kwargs.get("volume") is None: return await self._control_device(entity_id, action) - return await self.device_controller.control_device(entity_id, action, **kwargs) + data = {"entity_id": entity_id, "volume_level": kwargs.get("volume")} + return await self.device_controller.control_device(entity_id, action, **data) async def _control_climate(self, entity_id: str, action: str, **kwargs) -> Any: - self.ten_env.log_info(f"control_climate entity_id: {entity_id}, action: {action}, kwargs: {kwargs}") if kwargs.get("temperature") is None: return await self._control_device(entity_id, action) return await self.device_controller.control_device(entity_id, action, **kwargs) async def _control_fan(self, entity_id: str, action: str, **kwargs) -> Any: - self.ten_env.log_info(f"control_fan entity_id: {entity_id}, action: {action}, kwargs: {kwargs}") if kwargs.get("percentage") is None: return await self._control_device(entity_id, action)