diff --git a/Autotests/unit/test_llm_budget.py b/Autotests/unit/test_llm_budget.py index 634f945bd..bf0a86d3e 100644 --- a/Autotests/unit/test_llm_budget.py +++ b/Autotests/unit/test_llm_budget.py @@ -55,7 +55,13 @@ def _load_modules(): openai_provider = _MODULES["openai_provider"] asione = _MODULES["asione"] -PROMPT = "You are an agent. :-:-:-: Write an empty line to /tmp/paths.txt" +PROMPT = "You are an agent. :-:-:-: (HUMAN-MSG: Write an empty line to /tmp/paths.txt)" +IDLE_PROMPT = "You are an agent. :-:-:-: " + + +@pytest.fixture(autouse=True) +def no_pending_limit_notice(): + llm._limit_notice_pending = False def chat_response(content, finish_reason, completion_tokens=6000, reasoning_tokens=6000): @@ -156,6 +162,45 @@ def test_asione_empty_reply_out_of_budget_is_explained(): assert sent_text(make_asione(create).chat(PROMPT)) == llm.LLM_EMPTY_RESPONSE_MESSAGE +@pytest.mark.parametrize("make, response", [ + (make_openrouter, lambda: chat_response("", "length")), + (make_asione, lambda: chat_response("", "length")), + (make_openai, lambda: responses_response("", "incomplete", "max_output_tokens")), +]) +@pytest.mark.parametrize("prompt", [IDLE_PROMPT, "You are an agent. :-:-:-: DO NOT RE-SEND OR SPAM!"]) +def test_empty_reply_out_of_budget_without_a_user_message_sends_nothing(make, response, prompt): + assert make(FakeCreate(response())).chat(prompt) == "" + + +def test_limit_notice_comes_once_after_a_step_without_an_answer(): + provider = make_openrouter(FakeCreate( + chat_response('(query "hash maps")', "length"), + chat_response("", "length"), + chat_response("", "length"), + )) + assert provider.chat(PROMPT) == '(query "hash maps")' + assert sent_text(provider.chat(IDLE_PROMPT)) == llm.LLM_EMPTY_RESPONSE_MESSAGE + assert provider.chat(IDLE_PROMPT) == "" + + +@pytest.mark.parametrize("answer", ['(send "hi")', 'send "hi"', 'Sure.\nsend "hi"', '(query "x")\n(send "hi")']) +def test_limit_notice_is_skipped_once_the_user_got_an_answer(answer): + provider = make_openrouter(FakeCreate( + chat_response(answer, "stop"), + chat_response("", "length"), + )) + assert provider.chat(PROMPT) == answer + assert provider.chat(IDLE_PROMPT) == "" + + +def test_every_user_message_cut_by_the_limit_gets_its_own_notice(): + provider = make_openrouter(FakeCreate(*(chat_response("", "length") for _ in range(4)))) + assert sent_text(provider.chat(PROMPT)) == llm.LLM_EMPTY_RESPONSE_MESSAGE + assert provider.chat(IDLE_PROMPT) == "" + assert sent_text(provider.chat(PROMPT)) == llm.LLM_EMPTY_RESPONSE_MESSAGE + assert provider.chat(IDLE_PROMPT) == "" + + @pytest.mark.parametrize("effort, max_tokens, want_budget, want_enabled", [ ("medium", 6000, 3000, True), ("high", 6000, 4800, True), diff --git a/docs/reference-skills-communication.md b/docs/reference-skills-communication.md index 8eabe2f4b..7670337e3 100644 --- a/docs/reference-skills-communication.md +++ b/docs/reference-skills-communication.md @@ -26,7 +26,6 @@ No meaningful return value. Used for its side effect. ``` ### Notes / Limits -- **Deduplication:** `send` silently drops the call if `message` is identical to the previous one (`&lastsend` state). Change the text to send a near-duplicate. - Channel selection is set at `initChannels` time via `commchannel`. --- diff --git a/providers/asione.py b/providers/asione.py index 371ecaca3..4173ccb4d 100644 --- a/providers/asione.py +++ b/providers/asione.py @@ -75,8 +75,7 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", llm._log_chat_completion(self._name, self._model_name, response) if not raw: logger.warning("LLM returned an empty response") - if finish_reason == "length": - raw = llm._llm_empty_response_command() + raw = llm._reply_or_limit_notice(content, raw, finish_reason == "length") resp = self._clean_text(raw) return resp except Exception as e: diff --git a/providers/lib_llm_ext.py b/providers/lib_llm_ext.py index 28590cb7a..a9f94c153 100644 --- a/providers/lib_llm_ext.py +++ b/providers/lib_llm_ext.py @@ -2,7 +2,7 @@ import openai from typing import Optional, Tuple, Dict, Any from config import config_get_by_key -from src.helper import quote_arg +from src.helper import quote_arg, split_command_blocks from src.logger import get_logger PROMPT_DELIMITER = ":-:-:-:" @@ -20,6 +20,7 @@ "reasoning and the final answer draw from the same token limit, so higher " "reasoning levels need a higher token limit." ) +_limit_notice_pending = False logger = get_logger(__name__) @@ -67,6 +68,28 @@ def _llm_empty_response_command() -> str: """ return f"(send {quote_arg(LLM_EMPTY_RESPONSE_MESSAGE)})" +def _sends_to_user(raw: str) -> bool: + text = raw.replace("_quote_", '"').replace("_newline_", "\n") + for block in split_command_blocks(text): + command = block.lstrip().lstrip("(").split(maxsplit=1) + if command and command[0].rstrip(")") == "send": + return True + return False + +def _reply_or_limit_notice(content: str, raw: str, cut_by_limit: bool) -> str: + global _limit_notice_pending + _, usermsg = _split_system_user(content) + if "HUMAN-MSG:" in usermsg: + _limit_notice_pending = True + if raw: + if _sends_to_user(raw): + _limit_notice_pending = False + return raw + if cut_by_limit and _limit_notice_pending: + _limit_notice_pending = False + return _llm_empty_response_command() + return raw + def _split_system_user(content: str) -> Tuple[str, str]: """ MeTTa sends: @@ -188,8 +211,7 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", _log_chat_completion(self._name, self._model_name, response) if not raw: logger.warning("LLM returned an empty response") - if finish_reason == "length": - raw = _llm_empty_response_command() + raw = _reply_or_limit_notice(content, raw, finish_reason == "length") resp = self._clean_text(raw) return resp except Exception as e: diff --git a/providers/openai.py b/providers/openai.py index 8d0513539..043cfb45d 100644 --- a/providers/openai.py +++ b/providers/openai.py @@ -62,8 +62,7 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", llm._log_responses_completion(self._name, self._model_name, response) if not raw: logger.warning("LLM returned an empty response") - if incomplete_reason == "max_output_tokens": - raw = llm._llm_empty_response_command() + raw = llm._reply_or_limit_notice(content, raw, incomplete_reason == "max_output_tokens") return self._clean_text(raw) except Exception as e: logger.exception(f"[OpenAIProviderImpl.chat]: Exception while communicating with LLM: {e}") diff --git a/src/channels.metta b/src/channels.metta index 6213fc3ba..664d7c869 100644 --- a/src/channels.metta +++ b/src/channels.metta @@ -21,9 +21,6 @@ (commChannelReceive)) ;Send a message to all communication channels: -!(change-state! &lastsend "") (= (send $msg) - (if (!= $msg (get-state &lastsend)) - (progn (change-state! &lastsend $msg) - (let $safemsg (string-replace $msg "\n" "\\n") - (let $temp (cut) (commChannelSend $safemsg)))) _)) + (let $safemsg (string-replace $msg "\n" "\\n") + (let $temp (cut) (commChannelSend $safemsg))))