Convert Markdown to WhatsApp formatting on outbound enqueue.

LLM replies keep **/#/tables in admin history; WhatsApp queue text is rewritten to *bold*, plain links, and bullet rows (disable with AIA_WHATSAPP_MARKDOWN_CONVERT=0).

Co-authored-by: Cursor <cursoragent@cursor.com>

runtime/extensions/whatsapp/__init__.py

@@ -6,13 +6,16 @@ from .api import (
     normalize_whatsapp_allow_from_entries,
     normalize_whatsapp_target,
 )
+from .formatting import markdown_to_whatsapp_text, prepare_whatsapp_outbound_text
 
 __all__ = [
     "WHATSAPP_LEGACY_OUTBOUND_SEND_DEP_KEYS",
     "is_whatsapp_group_jid",
     "is_whatsapp_user_target",
     "looks_like_whatsapp_target_id",
+    "markdown_to_whatsapp_text",
     "normalize_whatsapp_allow_from_entries",
     "normalize_whatsapp_target",
+    "prepare_whatsapp_outbound_text",
 ]
 

runtime/extensions/whatsapp/formatting.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+import os
+import re
+from typing import Callable
+
+
+def whatsapp_markdown_convert_enabled() -> bool:
+    raw = str(os.getenv("AIA_WHATSAPP_MARKDOWN_CONVERT") or "1").strip().lower()
+    return raw not in {"0", "false", "no", "off"}
+
+
+_FENCE_RE = re.compile(r"```([^\n`]*)\n?(.*?)```", re.DOTALL)
+_INLINE_CODE_RE = re.compile(r"`([^`\n]+)`")
+_LINK_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)|\[([^\]]+)\]\(([^)]+)\)")
+_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE)
+_BOLD_STARS_RE = re.compile(r"\*\*(.+?)\*\*", re.DOTALL)
+_BOLD_UNDERS_RE = re.compile(r"__(.+?)__", re.DOTALL)
+_STRIKE_RE = re.compile(r"~~(.+?)~~", re.DOTALL)
+_HR_RE = re.compile(r"^[ \t]{0,3}(?:-{3,}|\*{3,}|_{3,})[ \t]*$", re.MULTILINE)
+_TABLE_SEP_RE = re.compile(r"^\s*\|?(?:\s*:?-+:?\s*\|)+\s*:?-+:?\s*\|?\s*$")
+_TABLE_ROW_RE = re.compile(r"^\s*\|(.+)\|\s*$")
+
+
+def _protect_segments(
+    text: str,
+    pattern: re.Pattern[str],
+    *,
+    tag: str,
+    rebuild: Callable[[re.Match[str]], str] | None = None,
+) -> tuple[str, dict[str, str]]:
+    slots: dict[str, str] = {}
+
+    def _sub(m: re.Match[str]) -> str:
+        key = f"\x00{tag}{len(slots)}\x00"
+        slots[key] = rebuild(m) if rebuild else m.group(0)
+        return key
+
+    return pattern.sub(_sub, text), slots
+
+
+def _restore(text: str, slots: dict[str, str]) -> str:
+    out = text
+    # Restore longer keys first in case of nested numbering.
+    for key in sorted(slots.keys(), key=len, reverse=True):
+        out = out.replace(key, slots[key])
+    return out
+
+
+def _convert_link(m: re.Match[str]) -> str:
+    if m.group(1) is not None:  # image
+        alt = str(m.group(1) or "").strip()
+        url = str(m.group(2) or "").strip()
+        if alt and url:
+            return f"{alt}: {url}"
+        return alt or url
+    label = str(m.group(3) or "").strip()
+    url = str(m.group(4) or "").strip()
+    if not url:
+        return label
+    if not label or label == url:
+        return url
+    return f"{label}: {url}"
+
+
+def _convert_table_block(lines: list[str]) -> list[str]:
+    rows: list[list[str]] = []
+    for line in lines:
+        if _TABLE_SEP_RE.match(line):
+            continue
+        m = _TABLE_ROW_RE.match(line)
+        if not m:
+            # Loose pipe row without outer requirement.
+            if "|" in line:
+                cells = [c.strip() for c in line.strip().strip("|").split("|")]
+                if any(cells):
+                    rows.append(cells)
+            continue
+        cells = [c.strip() for c in m.group(1).split("|")]
+        if any(cells):
+            rows.append(cells)
+    if not rows:
+        return lines
+    headers = rows[0]
+    body = rows[1:] if len(rows) > 1 else []
+    out: list[str] = []
+    if not body:
+        # Header-only / single row: keep as one line.
+        out.append(" · ".join(c for c in headers if c))
+        return out
+    for row in body:
+        parts: list[str] = []
+        for i, cell in enumerate(row):
+            if not cell:
+                continue
+            key = headers[i] if i < len(headers) and headers[i] else f"c{i+1}"
+            parts.append(f"{key}: {cell}")
+        if parts:
+            out.append("- " + " · ".join(parts))
+    return out or [" · ".join(headers)]
+
+
+def _rewrite_tables(text: str) -> str:
+    lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
+    out: list[str] = []
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        if "|" in line and (
+            _TABLE_ROW_RE.match(line)
+            or (i + 1 < len(lines) and _TABLE_SEP_RE.match(lines[i + 1]))
+        ):
+            block = [line]
+            i += 1
+            while i < len(lines) and ("|" in lines[i] or _TABLE_SEP_RE.match(lines[i])):
+                block.append(lines[i])
+                i += 1
+            out.extend(_convert_table_block(block))
+            continue
+        out.append(line)
+        i += 1
+    return "\n".join(out)
+
+
+def markdown_to_whatsapp_text(text: str) -> str:
+    """
+    Convert common Markdown (LLM output) into WhatsApp native formatting.
+
+    WhatsApp uses *bold*, _italic_, ~strike~, ```monospace``` / `code`.
+    Standard Markdown **bold**, # headings, tables, and [label](url) are rewritten.
+
+    Note: single-asterisk Markdown italic is NOT rewritten to _italic_. That conflicts
+    with WhatsApp bold (*text*) and would corrupt already-correct WA messages.
+    """
+    raw = str(text or "")
+    if not raw.strip():
+        return raw
+    if not whatsapp_markdown_convert_enabled():
+        return raw
+
+    work = raw.replace("\r\n", "\n").replace("\r", "\n")
+
+    # 1) Protect fenced + inline code (restore as WhatsApp monospace/code).
+    def _fence_rebuild(m: re.Match[str]) -> str:
+        body = str(m.group(2) or "").strip("\n")
+        return f"```{body}```" if body else "``` ```"
+
+    work, fence_slots = _protect_segments(work, _FENCE_RE, tag="FENCE", rebuild=_fence_rebuild)
+    work, inline_slots = _protect_segments(
+        work,
+        _INLINE_CODE_RE,
+        tag="CODE",
+        rebuild=lambda m: f"`{m.group(1)}`",
+    )
+
+    # 2) Links / images before emphasis (URLs may contain underscores).
+    work = _LINK_RE.sub(_convert_link, work)
+
+    # 3) Horizontal rules + tables.
+    work = _HR_RE.sub("────", work)
+    work = _rewrite_tables(work)
+
+    # 4) Bold / headings / strike. Keep bold as placeholders until the end so we
+    # never mistake *bold* for Markdown italic.
+    bold_slots: dict[str, str] = {}
+
+    def _hold_bold(inner: str) -> str:
+        text_inner = str(inner or "").strip()
+        if not text_inner:
+            return ""
+        key = f"\x00BOLD{len(bold_slots)}\x00"
+        bold_slots[key] = f"*{text_inner}*"
+        return key
+
+    def _bold_sub(m: re.Match[str]) -> str:
+        held = _hold_bold(m.group(1))
+        return held if held else m.group(0)
+
+    work = _BOLD_STARS_RE.sub(_bold_sub, work)
+    work = _BOLD_UNDERS_RE.sub(_bold_sub, work)
+    work = _HEADING_RE.sub(lambda m: _hold_bold(m.group(2)) or m.group(0), work)
+    work = _STRIKE_RE.sub(lambda m: f"~{str(m.group(1) or '').strip()}~", work)
+
+    # 5) Restore protected segments. Do not map *italic* → _italic_ (WA collision).
+    work = _restore(work, bold_slots)
+    work = _restore(work, inline_slots)
+    work = _restore(work, fence_slots)
+
+    # Collapse 3+ blank lines from table/heading rewrites.
+    work = re.sub(r"\n{3,}", "\n\n", work)
+    return work.strip()
+
+
+def prepare_whatsapp_outbound_text(text: str) -> str:
+    """Public entry used by outbound enqueue paths."""
+    return markdown_to_whatsapp_text(text)
+
+
+__all__ = [
+    "markdown_to_whatsapp_text",
+    "prepare_whatsapp_outbound_text",
+    "whatsapp_markdown_convert_enabled",
+]

svc/persistence/sqlite_store.py

@@ -6495,6 +6495,15 @@ class SqliteStore(ScheduledJobStoreMixin):
 
         msg_id = uuid.uuid4().hex
         ts = utc_now_iso()
+        ch = str(channel or "").strip().lower()
+        body = str(text)
+        if ch in {"whatsapp", "wa"}:
+            try:
+                from runtime.extensions.whatsapp.formatting import prepare_whatsapp_outbound_text
+
+                body = prepare_whatsapp_outbound_text(body)
+            except Exception:
+                body = str(text)
         with self._connect() as conn:
             conn.execute(
                 """
@@ -6502,7 +6511,7 @@ class SqliteStore(ScheduledJobStoreMixin):
                     (id, tenant_id, channel, account_id, chat_id, text, status, source, created_at, error)
                 VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, '')
                 """,
-                (msg_id, str(tenant_id), str(channel), str(account_id), str(chat_id), str(text), str(source), ts),
+                (msg_id, str(tenant_id), str(channel), str(account_id), str(chat_id), body, str(source), ts),
             )
         return msg_id
 

tests/test_whatsapp_markdown_format.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from runtime.extensions.whatsapp.formatting import markdown_to_whatsapp_text
+from svc.persistence.sqlite_store import SqliteStore
+
+
+class MarkdownToWhatsappTests(unittest.TestCase):
+    def test_bold_heading_link_strike(self) -> None:
+        src = (
+            "## Bandwidth summary\n"
+            "**Critical:** 3\n"
+            "See [UME](https://example.com/ume) for detail.\n"
+            "~~old~~ keep\n"
+        )
+        out = markdown_to_whatsapp_text(src)
+        self.assertIn("*Bandwidth summary*", out)
+        self.assertIn("*Critical:* 3", out)
+        self.assertIn("UME: https://example.com/ume", out)
+        self.assertIn("~old~", out)
+        self.assertNotIn("**", out)
+        self.assertNotIn("##", out)
+
+    def test_table_to_bullets(self) -> None:
+        src = (
+            "| Hostname | Util |\n"
+            "|---|---|\n"
+            "| BTM-A | 92% |\n"
+            "| ACH-B | 88% |\n"
+        )
+        out = markdown_to_whatsapp_text(src)
+        self.assertIn("- Hostname: BTM-A · Util: 92%", out)
+        self.assertIn("- Hostname: ACH-B · Util: 88%", out)
+        self.assertNotIn("|---|", out)
+
+    def test_preserves_code_and_list_markers(self) -> None:
+        src = (
+            "Run `show interface`:\n"
+            "```\n"
+            " GigabitEthernet0/1\n"
+            "```\n"
+            "- first\n"
+            "* second\n"
+            "Note: _careful_ word\n"
+        )
+        out = markdown_to_whatsapp_text(src)
+        self.assertIn("`show interface`", out)
+        self.assertIn("```", out)
+        self.assertIn("GigabitEthernet0/1", out)
+        self.assertIn("- first", out)
+        self.assertIn("* second", out)
+        self.assertIn("_careful_", out)
+
+    def test_already_whatsapp_bold_stays(self) -> None:
+        src = "*Critical:* 2 devices confirmed"
+        out = markdown_to_whatsapp_text(src)
+        self.assertEqual(out, src)
+
+    def test_disable_env(self) -> None:
+        prev = os.environ.get("AIA_WHATSAPP_MARKDOWN_CONVERT")
+        os.environ["AIA_WHATSAPP_MARKDOWN_CONVERT"] = "0"
+        try:
+            src = "**bold** and ## Head"
+            self.assertEqual(markdown_to_whatsapp_text(src), src)
+        finally:
+            if prev is None:
+                os.environ.pop("AIA_WHATSAPP_MARKDOWN_CONVERT", None)
+            else:
+                os.environ["AIA_WHATSAPP_MARKDOWN_CONVERT"] = prev
+
+
+class EnqueueConvertsWhatsappTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
+        self.db = Path(self._tmp.name) / "wa.sqlite"
+        os.environ["OPS_ASSISTANT_DB_PATH"] = str(self.db)
+        self.store = SqliteStore(str(self.db))
+
+    def tearDown(self) -> None:
+        self._tmp.cleanup()
+
+    def test_enqueue_whatsapp_converts_markdown(self) -> None:
+        msg_id = self.store.enqueue_channel_outbound_message(
+            channel="whatsapp",
+            chat_id="x@g.us",
+            text="## Done\n**ok**",
+            account_id="wa-default",
+        )
+        rows = self.store.list_pending_channel_outbound_messages(
+            channel="whatsapp", account_id="wa-default", limit=5
+        )
+        hit = next(r for r in rows if r["id"] == msg_id)
+        self.assertIn("*Done*", hit["text"])
+        self.assertIn("*ok*", hit["text"])
+        self.assertNotIn("**", hit["text"])
+
+    def test_enqueue_weixin_unchanged(self) -> None:
+        msg_id = self.store.enqueue_channel_outbound_message(
+            channel="weixin",
+            chat_id="wx-1",
+            text="**keep markdown**",
+            account_id="wx-default",
+        )
+        # list_pending is channel-specific; read via SQL-ish helper path
+        rows = self.store.list_pending_channel_outbound_messages(
+            channel="weixin", account_id="wx-default", limit=5
+        )
+        hit = next(r for r in rows if r["id"] == msg_id)
+        self.assertEqual(hit["text"], "**keep markdown**")
+
+
+if __name__ == "__main__":
+    unittest.main()