diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 8ac39bce..aaf78fcc 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -11,6 +11,7 @@ on: - "crates/bashkit-python/**" - "crates/bashkit/**" - "examples/*.py" + - "examples/*.ipynb" - "Cargo.toml" - "Cargo.lock" - ".github/workflows/python.yml" @@ -20,6 +21,7 @@ on: - "crates/bashkit-python/**" - "crates/bashkit/**" - "examples/*.py" + - "examples/*.ipynb" - "Cargo.toml" - "Cargo.lock" - ".github/workflows/python.yml" @@ -120,7 +122,7 @@ jobs: - name: Install local wheel run: | pip install bashkit --no-index --find-links crates/bashkit-python/dist --force-reinstall - pip install langchain-core langgraph fastapi httpx uvicorn + pip install langchain-core langgraph fastapi httpx uvicorn nbconvert jupyter-client ipykernel - name: Run examples run: | @@ -131,6 +133,11 @@ jobs: python crates/bashkit-python/examples/langgraph_async_tool.py python crates/bashkit-python/examples/fastapi_async_tool.py + - name: Run notebooks + run: | + jupyter nbconvert --to notebook --execute --ExecutePreprocessor.timeout=120 \ + examples/jupyter_basics.ipynb --output /tmp/jupyter_basics_out.ipynb + # Verify wheel builds and passes twine check build-wheel: name: Build wheel diff --git a/crates/bashkit-python/bashkit/_bashkit.pyi b/crates/bashkit-python/bashkit/_bashkit.pyi index 88bb2769..91ea61b5 100644 --- a/crates/bashkit-python/bashkit/_bashkit.pyi +++ b/crates/bashkit-python/bashkit/_bashkit.pyi @@ -424,6 +424,10 @@ class Bash: Not supported when ``external_handler`` is configured — use ``execute()`` (async) instead. ``on_output`` must be synchronous. Async ``custom_builtins`` callbacks run on a private loop here. + When called from inside a running event loop (e.g. Jupyter / IPython), + callbacks are dispatched to a background thread with their own loop so + that asyncio's "cannot run while another loop is running" restriction + is not triggered. Example:: @@ -778,6 +782,10 @@ class BashTool: """Execute bash commands synchronously (blocking). Async ``custom_builtins`` callbacks run on a private loop here. + When called from inside a running event loop (e.g. Jupyter / IPython), + callbacks are dispatched to a background thread with their own loop so + that asyncio's "cannot run while another loop is running" restriction + is not triggered. ``on_output`` must be synchronous. diff --git a/crates/bashkit-python/examples/jupyter_async_test.ipynb b/crates/bashkit-python/examples/jupyter_async_test.ipynb new file mode 100644 index 00000000..8cf094b6 --- /dev/null +++ b/crates/bashkit-python/examples/jupyter_async_test.ipynb @@ -0,0 +1,641 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "563a9854", + "metadata": {}, + "source": [ + "# bashkit in Jupyter: async + custom_builtins compatibility\n", + "\n", + "Jupyter already owns an asyncio event loop. This notebook checks that:\n", + "- `execute_sync()` works (creates its own private loop, no conflict)\n", + "- `await execute()` runs on Jupyter's caller loop\n", + "- Async callbacks reach the correct loop in both modes\n", + "- `custom_builtins` (BuiltinContext) work with sync and async callbacks\n", + "- ContextVar propagation works in Jupyter" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c9c207fd", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.836371Z", + "iopub.status.busy": "2026-04-16T22:49:28.836113Z", + "iopub.status.idle": "2026-04-16T22:49:28.848264Z", + "shell.execute_reply": "2026-04-16T22:49:28.847318Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "bashkit imported OK\n", + "Jupyter event loop: <_UnixSelectorEventLoop running=True closed=False debug=False>\n" + ] + } + ], + "source": [ + "import asyncio\n", + "import contextvars\n", + "\n", + "from bashkit import Bash, BuiltinContext, ScriptedTool\n", + "\n", + "print(\"bashkit imported OK\")\n", + "print(f\"Jupyter event loop: {asyncio.get_event_loop()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "44928c58", + "metadata": {}, + "source": [ + "## 1. Basic sync execution\n", + "\n", + "`execute_sync()` creates a private internal loop — should not conflict with Jupyter's loop." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "44e330ae", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.851832Z", + "iopub.status.busy": "2026-04-16T22:49:28.851588Z", + "iopub.status.idle": "2026-04-16T22:49:28.856930Z", + "shell.execute_reply": "2026-04-16T22:49:28.855674Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sync execute OK: 'hello from bashkit'\n" + ] + } + ], + "source": [ + "bash = Bash()\n", + "result = bash.execute_sync(\"echo 'hello from bashkit'\")\n", + "assert result.exit_code == 0\n", + "assert result.stdout.strip() == \"hello from bashkit\"\n", + "print(\"sync execute OK:\", repr(result.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "75b75348", + "metadata": {}, + "source": [ + "## 2. Basic async execution\n", + "\n", + "Jupyter lets you `await` at the top level — `execute()` should run on Jupyter's loop." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "3ff0f378", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.859633Z", + "iopub.status.busy": "2026-04-16T22:49:28.859367Z", + "iopub.status.idle": "2026-04-16T22:49:28.869305Z", + "shell.execute_reply": "2026-04-16T22:49:28.867769Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "async execute OK: 'hello async'\n" + ] + } + ], + "source": [ + "bash = Bash()\n", + "result = await bash.execute(\"echo 'hello async'\")\n", + "assert result.exit_code == 0\n", + "assert result.stdout.strip() == \"hello async\"\n", + "print(\"async execute OK:\", repr(result.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "8c4b938e", + "metadata": {}, + "source": [ + "## 3. custom_builtins — sync callback (BuiltinContext)\n", + "\n", + "A Python function registered as a bash builtin via `custom_builtins=`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "bb8275f1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.872160Z", + "iopub.status.busy": "2026-04-16T22:49:28.871888Z", + "iopub.status.idle": "2026-04-16T22:49:28.878066Z", + "shell.execute_reply": "2026-04-16T22:49:28.876528Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sync builtin OK: 'hello Jupiter'\n" + ] + } + ], + "source": [ + "def greet(ctx: BuiltinContext) -> str:\n", + " name = ctx.argv[0] if ctx.argv else \"world\"\n", + " return f\"hello {name}\\n\"\n", + "\n", + "\n", + "bash = Bash(custom_builtins={\"greet\": greet})\n", + "result = bash.execute_sync(\"greet Jupiter\")\n", + "assert result.exit_code == 0\n", + "assert result.stdout.strip() == \"hello Jupiter\"\n", + "print(\"sync builtin OK:\", repr(result.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "8bb2e5b1", + "metadata": {}, + "source": [ + "## 4. custom_builtins — sync callback with pipeline\n", + "\n", + "Builtin receiving stdin from a pipe." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "72b4368a", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.880726Z", + "iopub.status.busy": "2026-04-16T22:49:28.880407Z", + "iopub.status.idle": "2026-04-16T22:49:28.885843Z", + "shell.execute_reply": "2026-04-16T22:49:28.884914Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sync builtin + pipe OK: 'HELLO PIPE'\n" + ] + } + ], + "source": [ + "def shout(ctx: BuiltinContext) -> str:\n", + " text = ctx.stdin or \"\"\n", + " return text.upper()\n", + "\n", + "\n", + "bash = Bash(custom_builtins={\"shout\": shout})\n", + "result = bash.execute_sync(\"echo 'hello pipe' | shout\")\n", + "assert result.exit_code == 0\n", + "assert \"HELLO PIPE\" in result.stdout\n", + "print(\"sync builtin + pipe OK:\", repr(result.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "99aa6f92", + "metadata": {}, + "source": [ + "## 5. custom_builtins — async callback via execute_sync()\n", + "\n", + "`execute_sync()` spawns its own private event loop to drive the async callback.\n", + "In Jupyter there's already a running loop — the private loop must be independent." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "bd552eab", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.889269Z", + "iopub.status.busy": "2026-04-16T22:49:28.889016Z", + "iopub.status.idle": "2026-04-16T22:49:28.896905Z", + "shell.execute_reply": "2026-04-16T22:49:28.895563Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "async builtin via execute_sync OK: 'async-hello Jupyter'\n" + ] + } + ], + "source": [ + "jupyter_loop = asyncio.get_event_loop()\n", + "\n", + "\n", + "async def async_greet(ctx: BuiltinContext) -> str:\n", + " await asyncio.sleep(0) # yield to loop\n", + " name = ctx.argv[0] if ctx.argv else \"world\"\n", + " return f\"async-hello {name}\\n\"\n", + "\n", + "\n", + "bash = Bash(custom_builtins={\"async_greet\": async_greet})\n", + "result = bash.execute_sync(\"async_greet Jupyter\")\n", + "assert result.exit_code == 0\n", + "assert result.stdout.strip() == \"async-hello Jupyter\"\n", + "print(\"async builtin via execute_sync OK:\", repr(result.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "4e125857", + "metadata": {}, + "source": [ + "## 6. custom_builtins — async callback via await execute()\n", + "\n", + "With `await execute()`, async callbacks run on Jupyter's own event loop." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b63fc56f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.899781Z", + "iopub.status.busy": "2026-04-16T22:49:28.899407Z", + "iopub.status.idle": "2026-04-16T22:49:28.906936Z", + "shell.execute_reply": "2026-04-16T22:49:28.905203Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "async builtin via await execute uses caller loop: OK\n" + ] + } + ], + "source": [ + "jupyter_loop = asyncio.get_running_loop()\n", + "\n", + "captured_loop = None\n", + "\n", + "\n", + "async def loop_checker(ctx: BuiltinContext) -> str:\n", + " global captured_loop\n", + " captured_loop = asyncio.get_running_loop()\n", + " return \"checked\\n\"\n", + "\n", + "\n", + "bash = Bash(custom_builtins={\"loop_checker\": loop_checker})\n", + "result = await bash.execute(\"loop_checker\")\n", + "\n", + "assert result.exit_code == 0\n", + "assert captured_loop is jupyter_loop, \"async callback must run on Jupyter's loop\"\n", + "print(\"async builtin via await execute uses caller loop: OK\")" + ] + }, + { + "cell_type": "markdown", + "id": "ee5adc93", + "metadata": {}, + "source": [ + "## 7. ContextVar propagation — sync callback\n", + "\n", + "ContextVars set in the Jupyter cell propagate into builtin callbacks." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a5cbb381", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.909959Z", + "iopub.status.busy": "2026-04-16T22:49:28.909567Z", + "iopub.status.idle": "2026-04-16T22:49:28.915702Z", + "shell.execute_reply": "2026-04-16T22:49:28.914529Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ContextVar propagation (sync builtin) OK: 'req=jupyter-req-123'\n" + ] + } + ], + "source": [ + "request_id: contextvars.ContextVar[str] = contextvars.ContextVar(\"request_id\")\n", + "request_id.set(\"jupyter-req-123\")\n", + "\n", + "\n", + "def check_ctx(ctx: BuiltinContext) -> str:\n", + " rid = request_id.get(\"MISSING\")\n", + " return f\"req={rid}\\n\"\n", + "\n", + "\n", + "bash = Bash(custom_builtins={\"check_ctx\": check_ctx})\n", + "result = bash.execute_sync(\"check_ctx\")\n", + "assert result.exit_code == 0\n", + "assert result.stdout.strip() == \"req=jupyter-req-123\"\n", + "print(\"ContextVar propagation (sync builtin) OK:\", repr(result.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "c87a4db8", + "metadata": {}, + "source": [ + "## 8. ContextVar propagation — async callback via await execute()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "215ce9a2", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.918767Z", + "iopub.status.busy": "2026-04-16T22:49:28.918410Z", + "iopub.status.idle": "2026-04-16T22:49:28.925928Z", + "shell.execute_reply": "2026-04-16T22:49:28.924357Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ContextVar propagation (async builtin + await execute) OK: 'req=jupyter-req-456'\n" + ] + } + ], + "source": [ + "request_id.set(\"jupyter-req-456\")\n", + "\n", + "\n", + "async def async_check_ctx(ctx: BuiltinContext) -> str:\n", + " rid = request_id.get(\"MISSING\")\n", + " return f\"req={rid}\\n\"\n", + "\n", + "\n", + "bash = Bash(custom_builtins={\"async_check_ctx\": async_check_ctx})\n", + "result = await bash.execute(\"async_check_ctx\")\n", + "assert result.exit_code == 0\n", + "assert result.stdout.strip() == \"req=jupyter-req-456\"\n", + "print(\"ContextVar propagation (async builtin + await execute) OK:\", repr(result.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "8414c3e3", + "metadata": {}, + "source": [ + "## 9. ScriptedTool with custom tools in Jupyter\n", + "\n", + "`ScriptedTool.add_tool()` registers Python callbacks (both sync and async)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "5df2b146", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.929158Z", + "iopub.status.busy": "2026-04-16T22:49:28.928841Z", + "iopub.status.idle": "2026-04-16T22:49:28.940023Z", + "shell.execute_reply": "2026-04-16T22:49:28.938514Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ScriptedTool sync callback OK: 'fedcba'\n", + "ScriptedTool async callback via execute_sync OK: 'data:mykey'\n" + ] + } + ], + "source": [ + "def reverse(params, stdin=None):\n", + " text = params.get(\"text\", stdin or \"\")\n", + " return text[::-1] + \"\\n\"\n", + "\n", + "\n", + "async def fetch_data(params, stdin=None):\n", + " await asyncio.sleep(0) # simulate async I/O\n", + " key = params.get(\"key\", \"default\")\n", + " return f\"data:{key}\\n\"\n", + "\n", + "\n", + "tool = ScriptedTool(\"jupyter-test\")\n", + "tool.add_tool(\n", + " \"reverse\",\n", + " \"Reverse a string\",\n", + " callback=reverse,\n", + " schema={\"type\": \"object\", \"properties\": {\"text\": {\"type\": \"string\"}}},\n", + ")\n", + "tool.add_tool(\n", + " \"fetch_data\",\n", + " \"Fetch data by key\",\n", + " callback=fetch_data,\n", + " schema={\"type\": \"object\", \"properties\": {\"key\": {\"type\": \"string\"}}},\n", + ")\n", + "\n", + "r = tool.execute_sync(\"reverse --text abcdef\")\n", + "assert r.exit_code == 0\n", + "assert r.stdout.strip() == \"fedcba\"\n", + "print(\"ScriptedTool sync callback OK:\", repr(r.stdout.strip()))\n", + "\n", + "r = tool.execute_sync(\"fetch_data --key mykey\")\n", + "assert r.exit_code == 0\n", + "assert r.stdout.strip() == \"data:mykey\"\n", + "print(\"ScriptedTool async callback via execute_sync OK:\", repr(r.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "19fd0e79", + "metadata": {}, + "source": [ + "## 10. ScriptedTool — async callback via await execute() in Jupyter" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "2c706b1c", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.943444Z", + "iopub.status.busy": "2026-04-16T22:49:28.943157Z", + "iopub.status.idle": "2026-04-16T22:49:28.949243Z", + "shell.execute_reply": "2026-04-16T22:49:28.947960Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ScriptedTool async callback via await execute OK: 'data:asynckey'\n" + ] + } + ], + "source": [ + "r = await tool.execute(\"fetch_data --key asynckey\")\n", + "assert r.exit_code == 0\n", + "assert r.stdout.strip() == \"data:asynckey\"\n", + "print(\"ScriptedTool async callback via await execute OK:\", repr(r.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "bb10c09f", + "metadata": {}, + "source": [ + "## 11. Bash pipeline with custom builtin\n", + "\n", + "Custom builtins participate in shell pipelines." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "ce99a7d7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.952487Z", + "iopub.status.busy": "2026-04-16T22:49:28.952182Z", + "iopub.status.idle": "2026-04-16T22:49:28.958641Z", + "shell.execute_reply": "2026-04-16T22:49:28.957285Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Pipeline with custom builtin OK:\n", + "[INFO] line1\n", + "[INFO] line2\n", + "[INFO] line3\n" + ] + } + ], + "source": [ + "def tag(ctx: BuiltinContext) -> str:\n", + " label = ctx.argv[0] if ctx.argv else \"tag\"\n", + " text = ctx.stdin or \"\"\n", + " return \"\\n\".join(f\"[{label}] {line}\" for line in text.splitlines()) + \"\\n\"\n", + "\n", + "\n", + "bash = Bash(custom_builtins={\"tag\": tag})\n", + "result = bash.execute_sync(\"printf 'line1\\nline2\\nline3\\n' | tag INFO\")\n", + "assert result.exit_code == 0\n", + "assert \"[INFO] line1\" in result.stdout\n", + "assert \"[INFO] line3\" in result.stdout\n", + "print(\"Pipeline with custom builtin OK:\")\n", + "print(result.stdout.strip())" + ] + }, + { + "cell_type": "markdown", + "id": "325ee5aa", + "metadata": {}, + "source": [ + "## 12. Persistent state across cells\n", + "\n", + "VFS and shell variables persist across `execute_sync` calls on the same `Bash` instance." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "5672c2e0", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-16T22:49:28.961126Z", + "iopub.status.busy": "2026-04-16T22:49:28.960925Z", + "iopub.status.idle": "2026-04-16T22:49:28.966133Z", + "shell.execute_reply": "2026-04-16T22:49:28.964782Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Shell state persists across calls OK: 'hello'\n" + ] + } + ], + "source": [ + "bash = Bash()\n", + "bash.execute_sync(\"export MYVAR=hello\")\n", + "result = bash.execute_sync(\"echo $MYVAR\")\n", + "assert result.exit_code == 0\n", + "assert result.stdout.strip() == \"hello\"\n", + "print(\"Shell state persists across calls OK:\", repr(result.stdout.strip()))" + ] + }, + { + "cell_type": "markdown", + "id": "f9508d1b", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "All cells passed. Key findings:\n", + "- `execute_sync()` creates its own private event loop — no conflict with Jupyter's loop\n", + "- `await execute()` runs on Jupyter's caller loop; async callbacks see the same loop\n", + "- `custom_builtins` (sync and async, BuiltinContext) work in both execution modes\n", + "- ContextVar propagation into callbacks works from Jupyter cell scope\n", + "- `ScriptedTool` works as expected with sync and async callbacks" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/crates/bashkit-python/src/lib.rs b/crates/bashkit-python/src/lib.rs index 5a3565a9..73ad2177 100644 --- a/crates/bashkit-python/src/lib.rs +++ b/crates/bashkit-python/src/lib.rs @@ -1501,12 +1501,15 @@ enum PySyncLoopMode { struct PyPrivateAsyncLoop { event_loop: StdMutex>>, + // Cached Python helper for background-thread fallback (Jupyter/IPython compatibility). + bg_thread_runner: StdMutex>>, } impl PyPrivateAsyncLoop { fn new() -> Arc { Arc::new(Self { event_loop: StdMutex::new(None), + bg_thread_runner: StdMutex::new(None), }) } @@ -1522,7 +1525,66 @@ impl PyPrivateAsyncLoop { .clone_ref(py)) } - fn run_awaitable(&self, py: Python<'_>, awaitable: &Py) -> PyResult> { + fn bg_thread_runner(&self, py: Python<'_>) -> PyResult> { + let mut runner = self.bg_thread_runner.lock().expect("bg thread runner lock"); + if runner.is_none() { + // Spawns a daemon thread with a fresh event loop, runs the coroutine inside the + // supplied context so ContextVars propagate correctly, then joins and returns the + // result. The join() releases the GIL so the child thread can acquire it. + *runner = Some( + pyo3::types::PyModule::from_code( + py, + c"import asyncio, threading +def _run(coro, ctx): + r = [None]; e = [None] + def w(): + loop = asyncio.new_event_loop() + try: + r[0] = ctx.run(loop.run_until_complete, coro) + except BaseException as ex: + e[0] = ex + finally: + loop.close() + t = threading.Thread(target=w, daemon=True) + t.start() + t.join() + if e[0] is not None: + raise e[0] + return r[0]", + c"", + c"_bashkit_bg_loop", + )? + .getattr("_run")? + .unbind(), + ); + } + Ok(runner.as_ref().expect("runner prepared").clone_ref(py)) + } + + // `context` is the captured ContextVar snapshot; needed for the background-thread + // path so that Tasks inherit the correct ContextVars (background threads start with + // an empty context, unlike the calling thread whose ContextVars are already set). + fn run_awaitable( + &self, + py: Python<'_>, + awaitable: &Py, + context: &Py, + ) -> PyResult> { + // When a loop is already running on this thread (e.g. Jupyter / IPython), + // asyncio forbids run_until_complete on any loop — even a brand-new one. + // Fall back to a background thread that owns its own fresh event loop. + if py + .import("asyncio")? + .call_method0("get_running_loop") + .is_ok() + { + return self + .bg_thread_runner(py)? + .bind(py) + .call1((awaitable.bind(py), context.bind(py))) + .map(|v| v.unbind()); + } + self.event_loop(py)? .bind(py) .call_method1("run_until_complete", (awaitable.bind(py),)) @@ -1637,7 +1699,9 @@ impl PyCallbackSession { py: Python<'_>, awaitable: &Py, ) -> PyResult> { - self.private_async_loop.run_awaitable(py, awaitable) + let context = self.current_context(py); + self.private_async_loop + .run_awaitable(py, awaitable, &context) } } diff --git a/crates/bashkit-python/tests/test_jupyter_compat.py b/crates/bashkit-python/tests/test_jupyter_compat.py new file mode 100644 index 00000000..0f2e417f --- /dev/null +++ b/crates/bashkit-python/tests/test_jupyter_compat.py @@ -0,0 +1,220 @@ +"""Jupyter / IPython compatibility: execute_sync + async callbacks inside a running loop. + +Jupyter runs each cell inside a persistent asyncio event loop. This means: + - ``await execute()`` works natively (Jupyter accepts top-level await) + - ``execute_sync()`` must work even though asyncio forbids a second + ``run_until_complete`` on the same thread while a loop is running + - async ``custom_builtins`` / ``add_tool`` callbacks must run to completion + despite having no free loop slot on the calling thread + +Two simulation patterns are used: + ``asyncio.run(cell())`` — boots a loop, then calls synchronous ``execute_sync`` + from inside it; the closest analogue of a live + Jupyter cell running on a persistent loop + ``@pytest.mark.asyncio`` — runs the test coroutine on pytest's managed loop, + identical to what Jupyter does for async cells + +Surfaces covered: Bash, BashTool (``custom_builtins``), ScriptedTool +(``add_tool``). +""" + +import asyncio +import contextvars +import gc + +import pytest + +from bashkit import Bash, BashTool, BuiltinContext, ScriptedTool + +trace_id: contextvars.ContextVar[str] = contextvars.ContextVar("trace_id") + + +@pytest.fixture(autouse=True) +def _gc(): + yield + gc.collect() + + +# =========================================================================== +# asyncio.run() simulation — explicit Jupyter analogue +# +# asyncio.run() creates a new event loop, runs the coroutine on it, then +# closes the loop. While the coroutine runs, the thread has a *running* loop, +# reproducing the same condition that Jupyter maintains across all cells. +# =========================================================================== + + +def test_bash_execute_sync_async_builtin_inside_asyncio_run(): + """execute_sync() with async custom_builtin works inside asyncio.run().""" + + async def greet(ctx: BuiltinContext) -> str: + await asyncio.sleep(0) + return f"hello {ctx.argv[0] if ctx.argv else 'world'}\n" + + async def jupyter_cell(): + bash = Bash(custom_builtins={"greet": greet}) + return bash.execute_sync("greet Jupyter") + + result = asyncio.run(jupyter_cell()) + assert result.exit_code == 0 + assert result.stdout.strip() == "hello Jupyter" + + +def test_bash_execute_sync_contextvar_inside_asyncio_run(): + """ContextVars set before execute_sync() reach async callbacks inside asyncio.run().""" + + async def report(ctx: BuiltinContext) -> str: + return f"trace={trace_id.get('none')}\n" + + async def jupyter_cell(): + trace_id.set("cell-run-42") + bash = Bash(custom_builtins={"report": report}) + return bash.execute_sync("report") + + result = asyncio.run(jupyter_cell()) + assert result.exit_code == 0 + assert result.stdout.strip() == "trace=cell-run-42" + + +def test_scripted_tool_execute_sync_async_callback_inside_asyncio_run(): + """ScriptedTool.execute_sync() with async callback works inside asyncio.run().""" + + async def fetch(params, stdin=None): + await asyncio.sleep(0) + return f"data:{params.get('key', '?')}\n" + + async def jupyter_cell(): + tool = ScriptedTool("demo") + tool.add_tool( + "fetch", + "Fetch by key", + callback=fetch, + schema={"type": "object", "properties": {"key": {"type": "string"}}}, + ) + return tool.execute_sync("fetch --key mykey") + + result = asyncio.run(jupyter_cell()) + assert result.exit_code == 0 + assert result.stdout.strip() == "data:mykey" + + +def test_bash_execute_sync_multiple_async_calls_inside_asyncio_run(): + """Multiple execute_sync() calls from the same 'cell' all succeed.""" + + call_count = 0 + + async def counter(ctx: BuiltinContext) -> str: + nonlocal call_count + await asyncio.sleep(0) + call_count += 1 + return f"call:{call_count}\n" + + async def jupyter_cell(): + bash = Bash(custom_builtins={"counter": counter}) + r1 = bash.execute_sync("counter") + r2 = bash.execute_sync("counter") + r3 = bash.execute_sync("counter") + return r1, r2, r3 + + r1, r2, r3 = asyncio.run(jupyter_cell()) + assert r1.stdout.strip() == "call:1" + assert r2.stdout.strip() == "call:2" + assert r3.stdout.strip() == "call:3" + + +# =========================================================================== +# @pytest.mark.asyncio — live running loop, exact Jupyter match +# =========================================================================== + + +@pytest.mark.parametrize("factory", [Bash, BashTool], ids=["bash", "bash_tool"]) +@pytest.mark.asyncio +async def test_execute_sync_async_builtin_live_loop(factory): + """execute_sync() with async builtin works while pytest's event loop is running.""" + + async def greet(ctx: BuiltinContext) -> str: + await asyncio.sleep(0) + return f"hello {ctx.argv[0] if ctx.argv else 'world'}\n" + + shell = factory(custom_builtins={"greet": greet}) + result = shell.execute_sync("greet Jupyter") + + assert result.exit_code == 0 + assert result.stdout.strip() == "hello Jupyter" + + +@pytest.mark.parametrize("factory", [Bash, BashTool], ids=["bash", "bash_tool"]) +@pytest.mark.asyncio +async def test_execute_sync_contextvar_live_loop(factory): + """ContextVars propagate into async builtin callbacks with a live loop present.""" + + async def report(ctx: BuiltinContext) -> str: + return f"trace={trace_id.get('none')}\n" + + trace_id.set("live-loop-req") + shell = factory(custom_builtins={"report": report}) + result = shell.execute_sync("report") + + assert result.exit_code == 0 + assert result.stdout.strip() == "trace=live-loop-req" + + +@pytest.mark.asyncio +async def test_scripted_tool_execute_sync_async_callback_live_loop(): + """ScriptedTool.execute_sync() async callback works with a live loop.""" + + async def fetch(params, stdin=None): + await asyncio.sleep(0) + return f"data:{params.get('key', '?')}\n" + + tool = ScriptedTool("demo") + tool.add_tool( + "fetch", + "Fetch by key", + callback=fetch, + schema={"type": "object", "properties": {"key": {"type": "string"}}}, + ) + result = tool.execute_sync("fetch --key live") + + assert result.exit_code == 0 + assert result.stdout.strip() == "data:live" + + +# =========================================================================== +# await execute() — natural async Jupyter cell +# =========================================================================== + + +@pytest.mark.parametrize("factory", [Bash, BashTool], ids=["bash", "bash_tool"]) +@pytest.mark.asyncio +async def test_await_execute_async_builtin_uses_caller_loop(factory): + """async builtin via await execute() runs on the caller's event loop.""" + + caller_loop = asyncio.get_running_loop() + captured: list = [] + + async def inspect(ctx: BuiltinContext) -> str: + captured.append(asyncio.get_running_loop()) + return "ok\n" + + shell = factory(custom_builtins={"inspect": inspect}) + result = await shell.execute("inspect") + + assert result.exit_code == 0 + assert captured == [caller_loop] + + +@pytest.mark.parametrize("factory", [Bash, BashTool], ids=["bash", "bash_tool"]) +@pytest.mark.asyncio +async def test_await_execute_contextvar(factory): + """ContextVars propagate into async builtins via await execute().""" + + async def report(ctx: BuiltinContext) -> str: + return f"trace={trace_id.get('none')}\n" + + trace_id.set("await-req") + shell = factory(custom_builtins={"report": report}) + result = await shell.execute("report") + + assert result.exit_code == 0 + assert result.stdout.strip() == "trace=await-req" diff --git a/examples/jupyter_basics.ipynb b/examples/jupyter_basics.ipynb new file mode 100644 index 00000000..60adf790 --- /dev/null +++ b/examples/jupyter_basics.ipynb @@ -0,0 +1,317 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "md-intro", + "metadata": {}, + "source": [ + "# bashkit in Jupyter\n", + "\n", + "bashkit gives you a safe, sandboxed bash environment that integrates naturally\n", + "with Jupyter's async model.\n", + "\n", + "- **`await bash.execute()`** — the idiomatic Jupyter form; callbacks run on the notebook's own event loop\n", + "- **`bash.execute_sync()`** — also works in Jupyter even though a loop is already running\n", + "- **`custom_builtins`** — Python functions (sync *or* async) registered as bash commands\n", + "- **ContextVars** — values set in a cell propagate automatically into callbacks\n", + "\n", + "```\n", + "pip install bashkit\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "setup", + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "import contextvars\n", + "\n", + "from bashkit import Bash, BuiltinContext, ScriptedTool\n", + "\n", + "print(f\"Loop running: {asyncio.get_event_loop().is_running()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "md-async", + "metadata": {}, + "source": [ + "## Async execution — the natural Jupyter way\n", + "\n", + "Jupyter supports top-level `await`, so `await bash.execute()` is the idiomatic\n", + "form. Async `custom_builtins` callbacks run on the notebook's own event loop." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "async-exec", + "metadata": {}, + "outputs": [], + "source": [ + "bash = Bash()\n", + "result = await bash.execute(\"\"\"\n", + " echo 'files created:'\n", + " for f in report data config; do\n", + " touch /workspace/$f.txt\n", + " echo \" /workspace/$f.txt\"\n", + " done\n", + "\"\"\")\n", + "print(result.stdout)" + ] + }, + { + "cell_type": "markdown", + "id": "md-sync", + "metadata": {}, + "source": [ + "## Sync execution also works\n", + "\n", + "`execute_sync()` works even though Jupyter's event loop is already running.\n", + "Internally it dispatches async callbacks to a background thread when needed,\n", + "so there is no conflict." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sync-exec", + "metadata": {}, + "outputs": [], + "source": [ + "result = bash.execute_sync(\"ls /workspace/\")\n", + "print(result.stdout)" + ] + }, + { + "cell_type": "markdown", + "id": "md-vfs", + "metadata": {}, + "source": [ + "## Persistent workspace across cells\n", + "\n", + "The virtual filesystem and shell variables survive across calls on the same\n", + "`Bash` instance — just like a real shell session." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "vfs-write", + "metadata": {}, + "outputs": [], + "source": [ + "bash.execute_sync(\"\"\"\n", + " printf 'name,score\\nalice,95\\nbob,87\\ncarol,92\\ndave,78\\n' > /workspace/scores.csv\n", + " export THRESHOLD=90\n", + "\"\"\")\n", + "print(\"Data written.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "vfs-read", + "metadata": {}, + "outputs": [], + "source": [ + "# Variables and files from the previous cell are still there\n", + "result = bash.execute_sync(\"awk -F, -v t=$THRESHOLD '$2 >= t' /workspace/scores.csv\")\n", + "print(\"High scorers (>= $THRESHOLD):\")\n", + "print(result.stdout)" + ] + }, + { + "cell_type": "markdown", + "id": "md-sync-builtin", + "metadata": {}, + "source": [ + "## Custom Python commands — sync\n", + "\n", + "Register any Python function as a bash command via `custom_builtins=`.\n", + "The callback receives a `BuiltinContext` with `argv`, `stdin`, `env`, and `cwd`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sync-builtin", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "def summarize(ctx: BuiltinContext) -> str:\n", + " \"\"\"Read CSV from stdin, return JSON summary.\"\"\"\n", + " lines = [l for l in (ctx.stdin or \"\").strip().splitlines() if \",\" in l]\n", + " header, *rows = lines\n", + " cols = header.split(\",\")\n", + " records = [dict(zip(cols, r.split(\",\"))) for r in rows]\n", + " values = [float(r[cols[1]]) for r in records]\n", + " return json.dumps({\n", + " \"count\": len(values),\n", + " \"avg\": round(sum(values) / len(values), 1),\n", + " \"max\": max(values),\n", + " \"min\": min(values),\n", + " }) + \"\\n\"\n", + "\n", + "bash = Bash(custom_builtins={\"summarize\": summarize})\n", + "bash.execute_sync(\"printf 'name,score\\nalice,95\\nbob,87\\ncarol,92\\ndave,78\\n' > /data/scores.csv\")\n", + "\n", + "result = bash.execute_sync(\"cat /data/scores.csv | summarize | jq .\")\n", + "print(result.stdout)" + ] + }, + { + "cell_type": "markdown", + "id": "md-async-builtin", + "metadata": {}, + "source": [ + "## Custom Python commands — async\n", + "\n", + "Callbacks can be `async def` — bashkit drives them to completion whether you\n", + "call `execute_sync()` or `await execute()`. This is useful for async I/O\n", + "like HTTP requests or database queries inside a bash pipeline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "async-builtin-sync", + "metadata": {}, + "outputs": [], + "source": "\n# Async callback used via execute_sync() — works fine in Jupyter\nasync def enrich(ctx: BuiltinContext) -> str:\n name = ctx.argv[0] if ctx.argv else \"unknown\"\n await asyncio.sleep(0) # replace with real async I/O\n metadata = {\"name\": name, \"team\": \"engineering\", \"active\": True}\n return json.dumps(metadata) + \"\\n\"\n\nbash = Bash(custom_builtins={\"enrich\": enrich})\n\nresult = bash.execute_sync(\"enrich alice | jq -r '.name'\")\nprint(result.stdout)\n" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "async-builtin-await", + "metadata": {}, + "outputs": [], + "source": [ + "# Same callback, this time via await execute() — callback runs on Jupyter's loop\n", + "caller_loop = asyncio.get_running_loop()\n", + "seen_loops: list = []\n", + "\n", + "async def enrich_and_record(ctx: BuiltinContext) -> str:\n", + " seen_loops.append(asyncio.get_running_loop() is caller_loop)\n", + " await asyncio.sleep(0)\n", + " name = ctx.argv[0] if ctx.argv else \"?\"\n", + " return json.dumps({\"name\": name, \"enriched\": True}) + \"\\n\"\n", + "\n", + "bash = Bash(custom_builtins={\"enrich\": enrich_and_record})\n", + "result = await bash.execute(\"enrich bob | jq -r '.name'\")\n", + "\n", + "print(result.stdout.strip())\n", + "print(f\"Callback ran on Jupyter's loop: {seen_loops[0]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "md-contextvar", + "metadata": {}, + "source": [ + "## ContextVar propagation\n", + "\n", + "Values stored in `contextvars.ContextVar` before calling `execute()` or\n", + "`execute_sync()` are automatically visible inside callbacks. This pattern\n", + "is useful for per-cell tracing, request IDs, or streaming writers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "contextvar", + "metadata": {}, + "outputs": [], + "source": [ + "cell_events: contextvars.ContextVar[list] = contextvars.ContextVar(\"cell_events\")\n", + "\n", + "# Attach a collector to the current cell's context\n", + "log: list = []\n", + "cell_events.set(log)\n", + "\n", + "async def traced_fetch(ctx: BuiltinContext) -> str:\n", + " key = ctx.argv[0] if ctx.argv else \"?\"\n", + " # Write to the collector without any explicit reference to `log`\n", + " cell_events.get().append(f\"fetch:{key}\")\n", + " await asyncio.sleep(0)\n", + " return f\"result:{key}\\n\"\n", + "\n", + "bash = Bash(custom_builtins={\"fetch\": traced_fetch})\n", + "result = await bash.execute(\"fetch alpha; fetch beta; fetch gamma\")\n", + "\n", + "print(result.stdout)\n", + "print(\"Captured events:\", log)" + ] + }, + { + "cell_type": "markdown", + "id": "md-scripted", + "metadata": {}, + "source": [ + "## ScriptedTool — multi-tool orchestration\n", + "\n", + "`ScriptedTool` lets you register several Python callbacks as distinct bash\n", + "commands and compose them in a single script. The same Jupyter compatibility\n", + "guarantees apply." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "scripted-tool", + "metadata": {}, + "outputs": [], + "source": [ + "def parse_csv(params, stdin=None):\n", + " \"\"\"Convert CSV stdin to JSON array.\"\"\"\n", + " lines = (stdin or \"\").strip().splitlines()\n", + " header, *rows = lines\n", + " cols = header.split(\",\")\n", + " return json.dumps([dict(zip(cols, r.split(\",\"))) for r in rows]) + \"\\n\"\n", + "\n", + "async def format_report(params, stdin=None):\n", + " \"\"\"Turn JSON array into a markdown table.\"\"\"\n", + " records = json.loads(stdin or \"[]\")\n", + " if not records:\n", + " return \"no data\\n\"\n", + " headers = list(records[0].keys())\n", + " sep = \" | \"\n", + " header_row = sep.join(headers)\n", + " divider = sep.join(\"-\" * len(h) for h in headers)\n", + " rows = [sep.join(r.get(h, \"\") for h in headers) for r in records]\n", + " await asyncio.sleep(0)\n", + " return \"\\n\".join([header_row, divider, *rows]) + \"\\n\"\n", + "\n", + "tool = ScriptedTool(\"report-builder\")\n", + "tool.add_tool(\"parse-csv\", \"Parse CSV to JSON\", callback=parse_csv)\n", + "tool.add_tool(\"format-report\", \"Format JSON as markdown table\", callback=format_report)\n", + "\n", + "result = await tool.execute(\"\"\"\n", + " printf 'name,score,grade\\nalice,95,A\\nbob,87,B\\ncarol,92,A\\n' \\\n", + " | parse-csv \\\n", + " | format-report\n", + "\"\"\")\n", + "print(result.stdout)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/specs/python-package.md b/specs/python-package.md index e8e360d6..f6240980 100644 --- a/specs/python-package.md +++ b/specs/python-package.md @@ -214,6 +214,51 @@ result.to_dict() # dict Returns dict with `name`, `description`, `args_schema` for LangChain integration. +### custom_builtins and Async Callbacks + +`Bash` and `BashTool` accept `custom_builtins={"name": callback}` where each +callback is `Callable[[BuiltinContext], str | Awaitable[str]]`. + +**Sync callbacks** are called directly under the session's captured `contextvars` +snapshot and return a string. + +**Async callbacks** are driven to completion by one of two mechanisms depending +on whether a running asyncio event loop is present on the calling thread: + +| Calling context | Mechanism | +|---|---| +| `await execute()` | Callback scheduled as a `Task` on the **caller's running loop** | +| `execute_sync()` — no running loop | Callback driven by a **private event loop** shared across calls on the same `Bash` instance | +| `execute_sync()` — running loop present (e.g. Jupyter / IPython) | Callback driven by a **background daemon thread** with its own fresh event loop | + +The background-thread path is activated via `asyncio.get_running_loop()`. If the +call succeeds (a loop is already running on the thread), the awaitable is dispatched +to a daemon thread whose `run_until_complete` call is wrapped in `context.run()` +so ContextVars propagate correctly despite the thread switch. The helper is +cached on the `PyPrivateAsyncLoop` to avoid repeated module compilation. + +**ContextVar propagation**: ContextVars set before `execute()` or +`execute_sync()` are captured at call time and replayed inside each callback +invocation regardless of which mechanism is used. + +```python +import asyncio, contextvars +from bashkit import Bash, BuiltinContext + +trace_id = contextvars.ContextVar("trace_id") +trace_id.set("req-42") + +async def fetch(ctx: BuiltinContext) -> str: + await asyncio.sleep(0) # simulate async I/O + return f"trace={trace_id.get()}\n" + +bash = Bash(custom_builtins={"fetch": fetch}) + +# Works in plain Python, asyncio.run(), Jupyter, or any async framework: +result = bash.execute_sync("fetch") # "trace=req-42" +result = await bash.execute("fetch") # same, callback runs on caller loop +``` + ## Optional Dependencies ``` @@ -227,18 +272,23 @@ pip install bashkit[dev] # + pytest, pytest-asyncio File: `.github/workflows/python.yml` -Runs on push to main and PRs (path-filtered to `crates/bashkit-python/`, `crates/bashkit/`, -`Cargo.toml`, `Cargo.lock`). +Runs on push to main and PRs (path-filtered to `crates/bashkit-python/`, +`crates/bashkit/`, `examples/*.py`, `examples/*.ipynb`, `Cargo.toml`, +`Cargo.lock`). ``` PR / push to main ├── lint (ruff check + ruff format --check) ├── test (maturin develop + pytest, Python 3.9/3.12/3.13) - ├── examples (build wheel + run crates/bashkit-python/examples/) + ├── examples (build wheel + run crates/bashkit-python/examples/ + │ + execute examples/*.ipynb via nbconvert) ├── build-wheel (maturin build + twine check) └── python-check (gate job for branch protection) ``` +Notebooks in `examples/` are executed with `jupyter nbconvert --execute +--ExecutePreprocessor.timeout=120`. A cell error fails the CI job. + ## Linting - **Linter/formatter**: [ruff](https://docs.astral.sh/ruff/) (config in `pyproject.toml`)