Update app.py

app.py

@@ -7,7 +7,7 @@ sys.modules["pyaudioop"] = audioop_mock
 import gradio as gr
 import modal
 from PIL import Image
-import io, datetime
+import io, datetime, base64
 from huggingface_hub import InferenceClient
 from reportlab.lib.pagesizes import A4
 from reportlab.lib import colors
@@ -27,24 +27,23 @@ except ImportError:
     print("gTTS not available")
 
 CLASS_NAMES = [
-    "Black Background", "Abdominal Wall", "Liver", "Gastrointestinal Tract",
-    "Fat", "Grasper", "Connective Tissue", "Blood", "Cystic Duct",
-    "L-hook Electrocautery", "Gallbladder", "Hepatic Vein", "Liver Ligament"
+    "Black Background","Abdominal Wall","Liver","Gastrointestinal Tract",
+    "Fat","Grasper","Connective Tissue","Blood","Cystic Duct",
+    "L-hook Electrocautery","Gallbladder","Hepatic Vein","Liver Ligament"
 ]
-DANGER_CLASSES = ["Hepatic Vein", "Cystic Duct", "Blood"]
+DANGER_CLASSES = ["Hepatic Vein","Cystic Duct","Blood"]
 
 LANGUAGES = {
-    "English": {"code": "en", "prompt": "Respond in English."},
-    "French":  {"code": "fr", "prompt": "Réponds en français."},
-    "Spanish": {"code": "es", "prompt": "Responde en español."},
-    "German":  {"code": "de", "prompt": "Antworte auf Deutsch."},
-    "Arabic":  {"code": "ar", "prompt": "أجب باللغة العربية."},
-    "Hindi":   {"code": "hi", "prompt": "हिंदी में उत्तर दें।"},
+    "English": {"code":"en","prompt":"Respond in English."},
+    "French":  {"code":"fr","prompt":"Réponds en français."},
+    "Spanish": {"code":"es","prompt":"Responde en español."},
+    "German":  {"code":"de","prompt":"Antworte auf Deutsch."},
+    "Arabic":  {"code":"ar","prompt":"أجب باللغة العربية."},
+    "Hindi":   {"code":"hi","prompt":"हिंदी में उत्तर दें।"},
 }
 
 last_result = {}
 chat_context = {}
-last_ai_reply = {"text": "", "lang": "en"}
 
 try:
     client = InferenceClient(model="meta-llama/Llama-3.1-8B-Instruct", token=os.environ.get("HF_TOKEN"))
@@ -61,16 +60,15 @@ def get_detector():
 def pil_to_bytes(img):
     buf = io.BytesIO(); img.save(buf, format="PNG"); return buf.getvalue()
 
-def text_to_speech(text, lang_code):
-    if not TTS_AVAILABLE or not text: return None
+def tts_to_b64(text, lang_code):
+    if not TTS_AVAILABLE or not text: return ""
     try:
         tts = gTTS(text=text, lang=lang_code, slow=False)
-        path = "/tmp/surgisight_reply.mp3"
-        tts.save(path)
-        return path
+        buf = io.BytesIO()
+        tts.write_to_fp(buf)
+        return base64.b64encode(buf.getvalue()).decode()
     except Exception as e:
-        print(f"TTS error: {e}")
-        return None
+        print(f"TTS error: {e}"); return ""
 
 def generate_suggested_questions(tissue_list):
     questions = []
@@ -85,6 +83,88 @@ def generate_suggested_questions(tissue_list):
     questions.append("What are common complications in laparoscopic cholecystectomy?")
     return questions[:3]
 
+def render_chat_html(messages, lang_code):
+    """Render chat messages as clean Perplexity-style HTML with inline TTS."""
+    if not messages:
+        return '<p style="color:#888;text-align:center;padding:32px 0;">Run analysis first, then ask questions.</p>'
+
+    items = []
+    for i, msg in enumerate(messages):
+        role = msg["role"]
+        text = msg["content"]
+        # Convert **bold** markdown
+        import re
+        text_html = re.sub(r'\*\*(.+?)\*\*', r'<strong>\1</strong>', text)
+        text_html = text_html.replace("\n\n", "</p><p>").replace("\n", "<br>")
+        text_html = f"<p>{text_html}</p>"
+
+        if role == "user":
+            items.append(f"""
+            <div style="display:flex;justify-content:flex-end;margin:12px 0 4px;">
+              <div style="background:#2563eb;color:#fff;border-radius:20px 20px 4px 20px;
+                          padding:10px 16px;max-width:72%;font-size:0.9rem;line-height:1.6;">
+                {text_html}
+              </div>
+            </div>""")
+        else:
+            # AI message — generate TTS inline as base64 data URI
+            audio_b64 = tts_to_b64(text, lang_code)
+            audio_tag = ""
+            speaker_btn = ""
+            if audio_b64:
+                audio_id = f"audio-{i}"
+                audio_tag = f'<audio id="{audio_id}" src="data:audio/mp3;base64,{audio_b64}" preload="auto"></audio>'
+                speaker_btn = f"""
+                <button
+                  onclick="(function(){{
+                    var a=document.getElementById('{audio_id}');
+                    var b=this;
+                    if(!a.paused){{a.pause();a.currentTime=0;b.textContent='🔊';return;}}
+                    document.querySelectorAll('audio').forEach(function(x){{x.pause();x.currentTime=0;}});
+                    document.querySelectorAll('.spk-btn').forEach(function(x){{x.textContent='🔊';}});
+                    a.play();b.textContent='⏸';
+                    a.onended=function(){{b.textContent='🔊';}};
+                  }}).call(this)"
+                  class="spk-btn"
+                  title="Listen"
+                  style="background:none;border:none;cursor:pointer;font-size:1rem;
+                         padding:2px 6px;border-radius:6px;color:#6b7280;
+                         transition:color 0.15s;flex-shrink:0;margin-top:2px;"
+                  onmouseover="this.style.color='#2563eb'"
+                  onmouseout="this.style.color='#6b7280'">🔊</button>"""
+
+            items.append(f"""
+            <div style="display:flex;justify-content:flex-start;margin:12px 0 4px;">
+              <div style="max-width:82%;">
+                <div style="font-size:0.75rem;color:#6b7280;margin-bottom:4px;font-weight:500;">
+                  SurgiSight AI
+                </div>
+                <div style="display:flex;align-items:flex-start;gap:8px;">
+                  <div style="font-size:0.9rem;line-height:1.7;color:var(--body-text-color,#e0e0e0);">
+                    {text_html}
+                  </div>
+                  {speaker_btn}
+                </div>
+                {audio_tag}
+              </div>
+            </div>""")
+
+    html = f"""
+    <div id="chat-scroll"
+         style="height:420px;overflow-y:auto;padding:16px 8px;">
+      {''.join(items)}
+      <div id="chat-end"></div>
+    </div>
+    <script>
+      (function(){{
+        var el = document.getElementById('chat-end');
+        if(el) el.scrollIntoView({{behavior:'smooth'}});
+      }})();
+    </script>
+    """
+    return html
+
+
 def generate_pdf(original_image, annotated_image, seen, alert, explanation):
     pdf_path = "/tmp/surgisight_report.pdf"
     doc = SimpleDocTemplate(pdf_path, pagesize=A4,
@@ -161,12 +241,17 @@ def generate_pdf(original_image, annotated_image, seen, alert, explanation):
     return pdf_path
 
 
+# In-memory chat history (list of {"role":..,"content":..})
+_chat_history = []
+
 def segment_image(input_image, conf_threshold=0.25):
-    global last_result, chat_context, last_ai_reply
+    global last_result, chat_context, _chat_history
+    _chat_history = []
     if input_image is None:
         return (None, "Upload a frame to begin analysis.", "——", "——",
                 gr.update(visible=False), gr.update(visible=False),
-                gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), [])
+                gr.update(visible=False), gr.update(visible=False), gr.update(visible=False),
+                render_chat_html([], "en"))
     detector = get_detector()
     result = detector.run.remote(pil_to_bytes(input_image), conf_threshold)
     annotated_image = Image.open(io.BytesIO(result["annotated_bytes"]))
@@ -200,25 +285,26 @@ def segment_image(input_image, conf_threshold=0.25):
     chat_context = {"tissue_list": tissue_list, "alert": alert}
     last_result = {"original": input_image, "annotated": annotated_image,
                    "seen": seen, "alert": alert, "explanation": explanation}
+    intro = (f"Analysis complete. Detected: **{', '.join(tissue_list) if tissue_list else 'no structures'}**.\n\n"
+             f"{explanation}\n\nAsk me anything about these structures or laparoscopic surgery.")
+    _chat_history = [{"role": "assistant", "content": intro}]
     suggested = generate_suggested_questions(tissue_list) if tissue_list else []
     q1 = suggested[0] if len(suggested) > 0 else ""
     q2 = suggested[1] if len(suggested) > 1 else ""
     q3 = suggested[2] if len(suggested) > 2 else ""
-    intro = (f"Analysis complete. Detected: **{', '.join(tissue_list) if tissue_list else 'no structures'}**.\n\n"
-             f"{explanation}\n\nAsk me anything about these structures or laparoscopic surgery.")
-    last_ai_reply = {"text": intro, "lang": "en"}
-    initial_chat = [{"role": "assistant", "content": intro}]
     return (annotated_image, summary, alert, explanation,
             gr.update(visible=True), gr.update(visible=True),
             gr.update(value=q1, visible=bool(q1)),
             gr.update(value=q2, visible=bool(q2)),
-            gr.update(value=q3, visible=bool(q3)), initial_chat)
+            gr.update(value=q3, visible=bool(q3)),
+            render_chat_html(_chat_history, "en"))
 
 
-def chat_response(message, history, language):
-    global chat_context, last_ai_reply
+def send_message(message, language):
+    global chat_context, _chat_history
     if not message.strip():
-        return history, "", gr.update(visible=False), gr.update(visible=False)
+        lang_cfg = LANGUAGES.get(language, LANGUAGES["English"])
+        return render_chat_html(_chat_history, lang_cfg["code"]), ""
     tissue_list = chat_context.get("tissue_list", [])
     alert       = chat_context.get("alert", "")
     lang_cfg    = LANGUAGES.get(language, LANGUAGES["English"])
@@ -227,31 +313,15 @@ def chat_response(message, history, language):
         + (f"Current frame detected: {', '.join(tissue_list)}. Safety status: {alert}. " if tissue_list else "")
         + "Answer concisely in 2-4 sentences. "
         + lang_cfg["prompt"])
-    messages = [{"role": "system", "content": system_prompt}]
-    for msg in history:
-        messages.append({"role": msg["role"], "content": msg["content"]})
-    messages.append({"role": "user", "content": message})
+    messages = [{"role": "system", "content": system_prompt}] + _chat_history + [{"role": "user", "content": message}]
     try:
         response = client.chat_completion(messages, max_tokens=200, temperature=0.5)
         reply = response.choices[0].message.content.strip()
     except Exception as e:
         reply = f"Error: {str(e)}"
-    new_history = history + [
-        {"role": "user", "content": message},
-        {"role": "assistant", "content": reply}
-    ]
-    last_ai_reply = {"text": reply, "lang": lang_cfg["code"]}
-    audio_path = text_to_speech(reply, lang_cfg["code"])
-    audio_update = gr.update(value=audio_path, visible=True) if audio_path else gr.update(visible=False)
-    return new_history, "", audio_update, gr.update(visible=True)
-
-
-def replay_last(language):
-    global last_ai_reply
-    lang_cfg = LANGUAGES.get(language, LANGUAGES["English"])
-    text = last_ai_reply.get("text", "")
-    audio_path = text_to_speech(text, lang_cfg["code"])
-    return gr.update(value=audio_path, visible=True) if audio_path else gr.update(visible=False)
+    _chat_history.append({"role": "user",      "content": message})
+    _chat_history.append({"role": "assistant", "content": reply})
+    return render_chat_html(_chat_history, lang_cfg["code"]), ""
 
 
 def export_pdf():
@@ -260,7 +330,6 @@ def export_pdf():
                         last_result["seen"], last_result["alert"], last_result["explanation"])
 
 
-# CSS passed to launch() in Gradio 6
 css = """
 .gradio-container { max-width: 1120px !important; margin: 0 auto !important; }
 .generating, .progress-text, .progress-bar-wrap, .eta-bar, .eta-text { display: none !important; }
@@ -270,48 +339,28 @@ css = """
   flex-direction:column; align-items:center; justify-content:center;
 }
 #surgi-overlay.active { display:flex !important; }
-.or-spinner {
-  width:48px; height:48px;
-  border:4px solid rgba(255,255,255,0.2);
-  border-top-color:#fff; border-radius:50%;
-  animation:spin 0.8s linear infinite; margin-bottom:16px;
-}
+.or-spinner { width:48px;height:48px;border:4px solid rgba(255,255,255,0.2);border-top-color:#fff;border-radius:50%;animation:spin 0.8s linear infinite;margin-bottom:16px; }
 @keyframes spin { to { transform:rotate(360deg) } }
-.or-text { color:#fff; font-size:1rem; font-weight:600; text-align:center; }
-.or-sub  { color:rgba(255,255,255,0.6); font-size:0.82rem; margin-top:6px; }
+.or-text { color:#fff;font-size:1rem;font-weight:600;text-align:center; }
+.or-sub  { color:rgba(255,255,255,0.6);font-size:0.82rem;margin-top:6px; }
 """
 
-# JS runs once on page load — safe place for scripts in Gradio 6
 overlay_js = """
 () => {
-  // Inject overlay HTML
   const div = document.createElement('div');
   div.id = 'surgi-overlay';
-  div.innerHTML = `
-    <div class="or-spinner"></div>
-    <div class="or-text">Analysing Surgical Frame…</div>
-    <div class="or-sub">YOLOv8 · Modal GPU · Llama 3.1</div>
-  `;
+  div.innerHTML = '<div class="or-spinner"></div><div class="or-text">Analysing Surgical Frame…</div><div class="or-sub">YOLOv8 · Modal GPU · Llama 3.1</div>';
   document.body.appendChild(div);
-
-  // Attach click listener to run button
   function attachBtn() {
     const btn = document.querySelector('#run-btn button');
-    if (btn) {
-      btn.addEventListener('click', () => div.classList.add('active'));
-    } else {
-      setTimeout(attachBtn, 500);
-    }
+    if (btn) { btn.addEventListener('click', () => div.classList.add('active')); }
+    else { setTimeout(attachBtn, 500); }
   }
   attachBtn();
-
-  // Hide overlay when output image appears
   new MutationObserver(() => {
     const img = document.querySelector('#output-frame img');
     if (img && img.src && img.src.length > 80) div.classList.remove('active');
-  }).observe(document.body, {childList:true, subtree:true, attributes:true, attributeFilter:['src']});
-
-  // Safety timeout
+  }).observe(document.body, {childList:true,subtree:true,attributes:true,attributeFilter:['src']});
   setTimeout(() => div.classList.remove('active'), 90000);
 }
 """
@@ -319,59 +368,50 @@ overlay_js = """
 with gr.Blocks(title="SurgiSight — Surgical AI", js=overlay_js) as demo:
 
     gr.Markdown("""
-# SurgiSight — Surgical Tissue Segmentation
-**AI-powered anatomy detection for laparoscopic training** · YOLOv26n-seg · Llama 3.1 8B · Modal GPU · CholecSeg8k (MICCAI 2020) · mAP50: 0.581
+# 🔬 SurgiSight — Surgical Tissue Segmentation
+**AI-powered anatomy detection for laparoscopic training** · YOLOv8n-seg · Llama 3.1 8B · Modal GPU · CholecSeg8k · mAP50: 0.581
 
-> Research prototype — No real patient data.
+> ⚠️ Research prototype — not a medical device. No real patient data.
 """)
 
     with gr.Row(equal_height=True):
         with gr.Column(scale=1):
             input_img   = gr.Image(type="pil", label="Input Frame", height=340, elem_id="input-frame")
-            conf_slider = gr.Slider(0.1, 0.9, value=0.25, step=0.05,
-                label="Detection Confidence Threshold",
-                info="Lower = more detections  ·  Higher = high-confidence only")
-            run_btn = gr.Button("▶  Run Analysis", variant="primary", size="lg", elem_id="run-btn")
+            conf_slider = gr.Slider(0.1, 0.9, value=0.25, step=0.05, label="Confidence Threshold")
+            run_btn     = gr.Button("▶  Run Analysis", variant="primary", size="lg", elem_id="run-btn")
         with gr.Column(scale=1):
-            output_img = gr.Image(type="pil", label="Segmented Output", height=340, elem_id="output-frame")
+            output_img  = gr.Image(type="pil", label="Segmented Output", height=340, elem_id="output-frame")
 
     with gr.Row():
         danger_box  = gr.Textbox(label="⚠ Safety Assessment",   lines=2, placeholder="Awaiting analysis…")
-        output_text = gr.Textbox(label="🔍 Detected Structures", lines=6, placeholder="Detection results will appear here…")
+        output_text = gr.Textbox(label="🔍 Detected Structures", lines=6, placeholder="Results will appear here…")
 
-    explain_box = gr.Textbox(label="📖 Anatomy Brief", lines=4,
-        placeholder="AI anatomy explanation will appear after analysis…")
+    explain_box = gr.Textbox(label="📖 Anatomy Brief", lines=4, placeholder="AI explanation will appear after analysis…")
 
     with gr.Row():
         pdf_btn = gr.Button("⬇  Export Full Report (PDF)", visible=False)
     pdf_output = gr.File(label="Download Report", visible=False)
 
+    # ── Chat ──────────────────────────────────────────────────────────────────
     with gr.Column(visible=False) as chat_section:
         gr.Markdown("---\n### 💬 AI Anatomy Consult")
 
         with gr.Row():
             lang_select = gr.Dropdown(
                 choices=list(LANGUAGES.keys()), value="English",
-                label="🌍 Response Language", scale=1, min_width=180)
-            replay_btn = gr.Button("🔊 Replay Last Reply", scale=0, min_width=180, variant="secondary")
+                label="🌍 Response Language", scale=0, min_width=200)
 
         with gr.Row():
             sq1 = gr.Button("", visible=False, size="sm")
             sq2 = gr.Button("", visible=False, size="sm")
             sq3 = gr.Button("", visible=False, size="sm")
 
-        chatbot = gr.Chatbot(height=340, show_label=False)
-
-        # Audio player — autoplay, appears after each reply
-        audio_out = gr.Audio(
-            label="🔊 AI Voice Reply",
-            visible=False,
-            autoplay=True,
-            format="mp3"
-        )
+        # Clean HTML chat display — no Gradio chatbot widget
+        chat_display = gr.HTML(value="", label="")
 
         with gr.Row():
-            chat_input = gr.Textbox(placeholder="Ask about anatomy, safety, or technique…",
+            chat_input = gr.Textbox(
+                placeholder="Ask about anatomy, safety, or technique…",
                 show_label=False, scale=5, container=False)
             send_btn = gr.Button("Send", variant="primary", scale=1)
 
@@ -382,26 +422,22 @@ with gr.Blocks(title="SurgiSight — Surgical AI", js=overlay_js) as demo:
 
     gr.Markdown("---\n*CholecSeg8k · MICCAI 2020 · No patient data · Built for Build Small Hackathon 2026*")
 
-    # Wiring
+    # ── Wiring ────────────────────────────────────────────────────────────────
     run_btn.click(fn=segment_image, inputs=[input_img, conf_slider],
         outputs=[output_img, output_text, danger_box, explain_box,
-                 pdf_btn, chat_section, sq1, sq2, sq3, chatbot])
+                 pdf_btn, chat_section, sq1, sq2, sq3, chat_display])
 
     pdf_btn.click(fn=export_pdf, outputs=[pdf_output]).then(
         fn=lambda: gr.update(visible=True), outputs=[pdf_output])
 
-    send_btn.click(fn=chat_response,
-        inputs=[chat_input, chatbot, lang_select],
-        outputs=[chatbot, chat_input, audio_out, replay_btn])
-    chat_input.submit(fn=chat_response,
-        inputs=[chat_input, chatbot, lang_select],
-        outputs=[chatbot, chat_input, audio_out, replay_btn])
-
-    replay_btn.click(fn=replay_last, inputs=[lang_select], outputs=[audio_out])
+    send_btn.click(fn=send_message, inputs=[chat_input, lang_select],
+        outputs=[chat_display, chat_input])
+    chat_input.submit(fn=send_message, inputs=[chat_input, lang_select],
+        outputs=[chat_display, chat_input])
 
-    sq1.click(fn=chat_response, inputs=[sq1, chatbot, lang_select], outputs=[chatbot, chat_input, audio_out, replay_btn])
-    sq2.click(fn=chat_response, inputs=[sq2, chatbot, lang_select], outputs=[chatbot, chat_input, audio_out, replay_btn])
-    sq3.click(fn=chat_response, inputs=[sq3, chatbot, lang_select], outputs=[chatbot, chat_input, audio_out, replay_btn])
+    sq1.click(fn=send_message, inputs=[sq1, lang_select], outputs=[chat_display, chat_input])
+    sq2.click(fn=send_message, inputs=[sq2, lang_select], outputs=[chat_display, chat_input])
+    sq3.click(fn=send_message, inputs=[sq3, lang_select], outputs=[chat_display, chat_input])
 
 if __name__ == "__main__":
     demo.launch(css=css)