From cc9a516cb43eb98e892efcd589b78c2e7c7ff876 Mon Sep 17 00:00:00 2001 From: oliver Date: Tue, 11 Aug 2026 00:16:02 +0800 Subject: [PATCH] Convert Markdown to WhatsApp formatting on outbound enqueue. LLM replies keep **/#/tables in admin history; WhatsApp queue text is rewritten to *bold*, plain links, and bullet rows (disable with AIA_WHATSAPP_MARKDOWN_CONVERT=0). Co-authored-by: Cursor --- runtime/extensions/whatsapp/__init__.py | 3 + runtime/extensions/whatsapp/formatting.py | 203 ++++++++++++++++++++++ svc/persistence/sqlite_store.py | 11 +- tests/test_whatsapp_markdown_format.py | 117 +++++++++++++ 4 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 runtime/extensions/whatsapp/formatting.py create mode 100644 tests/test_whatsapp_markdown_format.py diff --git a/runtime/extensions/whatsapp/__init__.py b/runtime/extensions/whatsapp/__init__.py index 1f7d2acc..8f535d41 100644 --- a/runtime/extensions/whatsapp/__init__.py +++ b/runtime/extensions/whatsapp/__init__.py @@ -6,13 +6,16 @@ from .api import ( normalize_whatsapp_allow_from_entries, normalize_whatsapp_target, ) +from .formatting import markdown_to_whatsapp_text, prepare_whatsapp_outbound_text __all__ = [ "WHATSAPP_LEGACY_OUTBOUND_SEND_DEP_KEYS", "is_whatsapp_group_jid", "is_whatsapp_user_target", "looks_like_whatsapp_target_id", + "markdown_to_whatsapp_text", "normalize_whatsapp_allow_from_entries", "normalize_whatsapp_target", + "prepare_whatsapp_outbound_text", ] diff --git a/runtime/extensions/whatsapp/formatting.py b/runtime/extensions/whatsapp/formatting.py new file mode 100644 index 00000000..114472f1 --- /dev/null +++ b/runtime/extensions/whatsapp/formatting.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import os +import re +from typing import Callable + + +def whatsapp_markdown_convert_enabled() -> bool: + raw = str(os.getenv("AIA_WHATSAPP_MARKDOWN_CONVERT") or "1").strip().lower() + return raw not in {"0", "false", "no", "off"} + + +_FENCE_RE = re.compile(r"```([^\n`]*)\n?(.*?)```", re.DOTALL) +_INLINE_CODE_RE = re.compile(r"`([^`\n]+)`") +_LINK_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)|\[([^\]]+)\]\(([^)]+)\)") +_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE) +_BOLD_STARS_RE = re.compile(r"\*\*(.+?)\*\*", re.DOTALL) +_BOLD_UNDERS_RE = re.compile(r"__(.+?)__", re.DOTALL) +_STRIKE_RE = re.compile(r"~~(.+?)~~", re.DOTALL) +_HR_RE = re.compile(r"^[ \t]{0,3}(?:-{3,}|\*{3,}|_{3,})[ \t]*$", re.MULTILINE) +_TABLE_SEP_RE = re.compile(r"^\s*\|?(?:\s*:?-+:?\s*\|)+\s*:?-+:?\s*\|?\s*$") +_TABLE_ROW_RE = re.compile(r"^\s*\|(.+)\|\s*$") + + +def _protect_segments( + text: str, + pattern: re.Pattern[str], + *, + tag: str, + rebuild: Callable[[re.Match[str]], str] | None = None, +) -> tuple[str, dict[str, str]]: + slots: dict[str, str] = {} + + def _sub(m: re.Match[str]) -> str: + key = f"\x00{tag}{len(slots)}\x00" + slots[key] = rebuild(m) if rebuild else m.group(0) + return key + + return pattern.sub(_sub, text), slots + + +def _restore(text: str, slots: dict[str, str]) -> str: + out = text + # Restore longer keys first in case of nested numbering. + for key in sorted(slots.keys(), key=len, reverse=True): + out = out.replace(key, slots[key]) + return out + + +def _convert_link(m: re.Match[str]) -> str: + if m.group(1) is not None: # image + alt = str(m.group(1) or "").strip() + url = str(m.group(2) or "").strip() + if alt and url: + return f"{alt}: {url}" + return alt or url + label = str(m.group(3) or "").strip() + url = str(m.group(4) or "").strip() + if not url: + return label + if not label or label == url: + return url + return f"{label}: {url}" + + +def _convert_table_block(lines: list[str]) -> list[str]: + rows: list[list[str]] = [] + for line in lines: + if _TABLE_SEP_RE.match(line): + continue + m = _TABLE_ROW_RE.match(line) + if not m: + # Loose pipe row without outer requirement. + if "|" in line: + cells = [c.strip() for c in line.strip().strip("|").split("|")] + if any(cells): + rows.append(cells) + continue + cells = [c.strip() for c in m.group(1).split("|")] + if any(cells): + rows.append(cells) + if not rows: + return lines + headers = rows[0] + body = rows[1:] if len(rows) > 1 else [] + out: list[str] = [] + if not body: + # Header-only / single row: keep as one line. + out.append(" · ".join(c for c in headers if c)) + return out + for row in body: + parts: list[str] = [] + for i, cell in enumerate(row): + if not cell: + continue + key = headers[i] if i < len(headers) and headers[i] else f"c{i+1}" + parts.append(f"{key}: {cell}") + if parts: + out.append("- " + " · ".join(parts)) + return out or [" · ".join(headers)] + + +def _rewrite_tables(text: str) -> str: + lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + out: list[str] = [] + i = 0 + while i < len(lines): + line = lines[i] + if "|" in line and ( + _TABLE_ROW_RE.match(line) + or (i + 1 < len(lines) and _TABLE_SEP_RE.match(lines[i + 1])) + ): + block = [line] + i += 1 + while i < len(lines) and ("|" in lines[i] or _TABLE_SEP_RE.match(lines[i])): + block.append(lines[i]) + i += 1 + out.extend(_convert_table_block(block)) + continue + out.append(line) + i += 1 + return "\n".join(out) + + +def markdown_to_whatsapp_text(text: str) -> str: + """ + Convert common Markdown (LLM output) into WhatsApp native formatting. + + WhatsApp uses *bold*, _italic_, ~strike~, ```monospace``` / `code`. + Standard Markdown **bold**, # headings, tables, and [label](url) are rewritten. + + Note: single-asterisk Markdown italic is NOT rewritten to _italic_. That conflicts + with WhatsApp bold (*text*) and would corrupt already-correct WA messages. + """ + raw = str(text or "") + if not raw.strip(): + return raw + if not whatsapp_markdown_convert_enabled(): + return raw + + work = raw.replace("\r\n", "\n").replace("\r", "\n") + + # 1) Protect fenced + inline code (restore as WhatsApp monospace/code). + def _fence_rebuild(m: re.Match[str]) -> str: + body = str(m.group(2) or "").strip("\n") + return f"```{body}```" if body else "``` ```" + + work, fence_slots = _protect_segments(work, _FENCE_RE, tag="FENCE", rebuild=_fence_rebuild) + work, inline_slots = _protect_segments( + work, + _INLINE_CODE_RE, + tag="CODE", + rebuild=lambda m: f"`{m.group(1)}`", + ) + + # 2) Links / images before emphasis (URLs may contain underscores). + work = _LINK_RE.sub(_convert_link, work) + + # 3) Horizontal rules + tables. + work = _HR_RE.sub("────", work) + work = _rewrite_tables(work) + + # 4) Bold / headings / strike. Keep bold as placeholders until the end so we + # never mistake *bold* for Markdown italic. + bold_slots: dict[str, str] = {} + + def _hold_bold(inner: str) -> str: + text_inner = str(inner or "").strip() + if not text_inner: + return "" + key = f"\x00BOLD{len(bold_slots)}\x00" + bold_slots[key] = f"*{text_inner}*" + return key + + def _bold_sub(m: re.Match[str]) -> str: + held = _hold_bold(m.group(1)) + return held if held else m.group(0) + + work = _BOLD_STARS_RE.sub(_bold_sub, work) + work = _BOLD_UNDERS_RE.sub(_bold_sub, work) + work = _HEADING_RE.sub(lambda m: _hold_bold(m.group(2)) or m.group(0), work) + work = _STRIKE_RE.sub(lambda m: f"~{str(m.group(1) or '').strip()}~", work) + + # 5) Restore protected segments. Do not map *italic* → _italic_ (WA collision). + work = _restore(work, bold_slots) + work = _restore(work, inline_slots) + work = _restore(work, fence_slots) + + # Collapse 3+ blank lines from table/heading rewrites. + work = re.sub(r"\n{3,}", "\n\n", work) + return work.strip() + + +def prepare_whatsapp_outbound_text(text: str) -> str: + """Public entry used by outbound enqueue paths.""" + return markdown_to_whatsapp_text(text) + + +__all__ = [ + "markdown_to_whatsapp_text", + "prepare_whatsapp_outbound_text", + "whatsapp_markdown_convert_enabled", +] diff --git a/svc/persistence/sqlite_store.py b/svc/persistence/sqlite_store.py index f8a9a761..8a430173 100644 --- a/svc/persistence/sqlite_store.py +++ b/svc/persistence/sqlite_store.py @@ -6495,6 +6495,15 @@ class SqliteStore(ScheduledJobStoreMixin): msg_id = uuid.uuid4().hex ts = utc_now_iso() + ch = str(channel or "").strip().lower() + body = str(text) + if ch in {"whatsapp", "wa"}: + try: + from runtime.extensions.whatsapp.formatting import prepare_whatsapp_outbound_text + + body = prepare_whatsapp_outbound_text(body) + except Exception: + body = str(text) with self._connect() as conn: conn.execute( """ @@ -6502,7 +6511,7 @@ class SqliteStore(ScheduledJobStoreMixin): (id, tenant_id, channel, account_id, chat_id, text, status, source, created_at, error) VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, '') """, - (msg_id, str(tenant_id), str(channel), str(account_id), str(chat_id), str(text), str(source), ts), + (msg_id, str(tenant_id), str(channel), str(account_id), str(chat_id), body, str(source), ts), ) return msg_id diff --git a/tests/test_whatsapp_markdown_format.py b/tests/test_whatsapp_markdown_format.py new file mode 100644 index 00000000..1e329246 --- /dev/null +++ b/tests/test_whatsapp_markdown_format.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import os +import tempfile +import unittest +from pathlib import Path + +from runtime.extensions.whatsapp.formatting import markdown_to_whatsapp_text +from svc.persistence.sqlite_store import SqliteStore + + +class MarkdownToWhatsappTests(unittest.TestCase): + def test_bold_heading_link_strike(self) -> None: + src = ( + "## Bandwidth summary\n" + "**Critical:** 3\n" + "See [UME](https://example.com/ume) for detail.\n" + "~~old~~ keep\n" + ) + out = markdown_to_whatsapp_text(src) + self.assertIn("*Bandwidth summary*", out) + self.assertIn("*Critical:* 3", out) + self.assertIn("UME: https://example.com/ume", out) + self.assertIn("~old~", out) + self.assertNotIn("**", out) + self.assertNotIn("##", out) + + def test_table_to_bullets(self) -> None: + src = ( + "| Hostname | Util |\n" + "|---|---|\n" + "| BTM-A | 92% |\n" + "| ACH-B | 88% |\n" + ) + out = markdown_to_whatsapp_text(src) + self.assertIn("- Hostname: BTM-A · Util: 92%", out) + self.assertIn("- Hostname: ACH-B · Util: 88%", out) + self.assertNotIn("|---|", out) + + def test_preserves_code_and_list_markers(self) -> None: + src = ( + "Run `show interface`:\n" + "```\n" + " GigabitEthernet0/1\n" + "```\n" + "- first\n" + "* second\n" + "Note: _careful_ word\n" + ) + out = markdown_to_whatsapp_text(src) + self.assertIn("`show interface`", out) + self.assertIn("```", out) + self.assertIn("GigabitEthernet0/1", out) + self.assertIn("- first", out) + self.assertIn("* second", out) + self.assertIn("_careful_", out) + + def test_already_whatsapp_bold_stays(self) -> None: + src = "*Critical:* 2 devices confirmed" + out = markdown_to_whatsapp_text(src) + self.assertEqual(out, src) + + def test_disable_env(self) -> None: + prev = os.environ.get("AIA_WHATSAPP_MARKDOWN_CONVERT") + os.environ["AIA_WHATSAPP_MARKDOWN_CONVERT"] = "0" + try: + src = "**bold** and ## Head" + self.assertEqual(markdown_to_whatsapp_text(src), src) + finally: + if prev is None: + os.environ.pop("AIA_WHATSAPP_MARKDOWN_CONVERT", None) + else: + os.environ["AIA_WHATSAPP_MARKDOWN_CONVERT"] = prev + + +class EnqueueConvertsWhatsappTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + self.db = Path(self._tmp.name) / "wa.sqlite" + os.environ["OPS_ASSISTANT_DB_PATH"] = str(self.db) + self.store = SqliteStore(str(self.db)) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def test_enqueue_whatsapp_converts_markdown(self) -> None: + msg_id = self.store.enqueue_channel_outbound_message( + channel="whatsapp", + chat_id="x@g.us", + text="## Done\n**ok**", + account_id="wa-default", + ) + rows = self.store.list_pending_channel_outbound_messages( + channel="whatsapp", account_id="wa-default", limit=5 + ) + hit = next(r for r in rows if r["id"] == msg_id) + self.assertIn("*Done*", hit["text"]) + self.assertIn("*ok*", hit["text"]) + self.assertNotIn("**", hit["text"]) + + def test_enqueue_weixin_unchanged(self) -> None: + msg_id = self.store.enqueue_channel_outbound_message( + channel="weixin", + chat_id="wx-1", + text="**keep markdown**", + account_id="wx-default", + ) + # list_pending is channel-specific; read via SQL-ish helper path + rows = self.store.list_pending_channel_outbound_messages( + channel="weixin", account_id="wx-default", limit=5 + ) + hit = next(r for r in rows if r["id"] == msg_id) + self.assertEqual(hit["text"], "**keep markdown**") + + +if __name__ == "__main__": + unittest.main()