Update README.md

README.md

@@ -87,11 +87,33 @@ processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
 ### Inference
 
 ```python
+def _build_unicode_to_bytes_map() -> dict[str, int]:
+    """Build inverse of GPT-2's bytes_to_unicode mapping."""
+    bs = (
+        list(range(ord("!"), ord("~") + 1))
+        + list(range(ord("¡"), ord("¬") + 1))
+        + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(256):
+        if b not in bs:
+            bs.append(b)
+            cs.append(256 + n)
+            n += 1
+    return {chr(c): b for b, c in zip(bs, cs)}
+
+_UNICODE_TO_BYTE = _build_unicode_to_bytes_map()
+
 def fix_byte_encoding(text: str) -> str:
-    """Fix byte-level BPE encoding for proper emoji/unicode display."""
+    """Fix byte-level BPE encoding for proper emoji/unicode display.
+
+    Example: "ðŁ¤Ĺ" -> "🤗"
+    """
     try:
-        return text.encode("latin-1").decode("utf-8")
-    except (UnicodeDecodeError, UnicodeEncodeError):
+        byte_values = bytes([_UNICODE_TO_BYTE.get(c, ord(c)) for c in text])
+        return byte_values.decode("utf-8")
+    except (UnicodeDecodeError, KeyError, ValueError):
         return text
 
 messages = [
@@ -119,7 +141,7 @@ response = fix_byte_encoding(response)  # Fix emoji/unicode display
 print(response)
 ```
 
-> **Note:** The `fix_byte_encoding` helper is needed because Mistral uses byte-level BPE tokenization. Without it, emojis and special characters may display incorrectly.
+> **Note:** The `fix_byte_encoding` helper is needed because Mistral uses byte-level BPE tokenization (GPT-2 style). UTF-8 bytes are encoded as individual Unicode characters (e.g., 🤗 becomes `ðŁ¤Ĺ`). This function reverses that mapping for proper display.
 
 ### Quick Test