Update app.py
Browse files
app.py
CHANGED
|
@@ -15,6 +15,11 @@ from reportlab.lib.units import cm
|
|
| 15 |
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image as RLImage, Table, TableStyle, HRFlowable
|
| 16 |
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
|
| 17 |
from reportlab.lib.enums import TA_CENTER
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
print("All imports OK")
|
| 20 |
|
|
@@ -33,7 +38,6 @@ CLASS_NAMES = [
|
|
| 33 |
]
|
| 34 |
DANGER_CLASSES = ["Hepatic Vein","Cystic Duct","Blood"]
|
| 35 |
|
| 36 |
-
# Only languages with reliable gTTS support
|
| 37 |
LANGUAGES = {
|
| 38 |
"English": {"code": "en", "prompt": "Respond in English."},
|
| 39 |
"French": {"code": "fr", "prompt": "RΓ©ponds en franΓ§ais."},
|
|
@@ -41,8 +45,6 @@ LANGUAGES = {
|
|
| 41 |
|
| 42 |
last_result = {}
|
| 43 |
chat_context = {}
|
| 44 |
-
|
| 45 |
-
# Each entry: {"role": "user"/"assistant", "en": "original english text", "display": "current display text"}
|
| 46 |
_chat_history = []
|
| 47 |
|
| 48 |
try:
|
|
@@ -71,12 +73,10 @@ def tts_to_b64(text, lang_code):
|
|
| 71 |
print(f"TTS error: {e}"); return ""
|
| 72 |
|
| 73 |
def translate_to(text, lang_cfg):
|
| 74 |
-
""
|
| 75 |
-
if lang_cfg["code"] == "en" or not client:
|
| 76 |
-
return text
|
| 77 |
try:
|
| 78 |
-
|
| 79 |
-
|
| 80 |
resp = client.chat_completion([{"role":"user","content":prompt}], max_tokens=300, temperature=0.1)
|
| 81 |
return resp.choices[0].message.content.strip()
|
| 82 |
except:
|
|
@@ -96,19 +96,15 @@ def generate_suggested_questions(tissue_list):
|
|
| 96 |
return questions[:3]
|
| 97 |
|
| 98 |
def render_chat_html(history, lang_code):
|
| 99 |
-
"""Render Perplexity-style chat with π per AI message. Audio embedded as base64."""
|
| 100 |
if not history:
|
| 101 |
return '<p style="color:#888;text-align:center;padding:32px 0;font-size:0.9rem;">Run analysis first, then ask questions here.</p>'
|
| 102 |
-
|
| 103 |
items = []
|
| 104 |
for i, msg in enumerate(history):
|
| 105 |
role = msg["role"]
|
| 106 |
text = msg["display"]
|
| 107 |
-
# Markdown bold
|
| 108 |
text_html = re.sub(r'\*\*(.+?)\*\*', r'<strong>\1</strong>', text)
|
| 109 |
text_html = text_html.replace("\n\n", "</p><p style='margin:6px 0;'>").replace("\n", "<br>")
|
| 110 |
text_html = f"<p style='margin:0;'>{text_html}</p>"
|
| 111 |
-
|
| 112 |
if role == "user":
|
| 113 |
items.append(f"""
|
| 114 |
<div style="display:flex;justify-content:flex-end;margin:14px 0 2px;">
|
|
@@ -136,7 +132,6 @@ def render_chat_html(history, lang_code):
|
|
| 136 |
f'border-radius:6px;color:#6b7280;flex-shrink:0;margin-top:1px;transition:color 0.15s;" '
|
| 137 |
f'onmouseover="this.style.color=\'#2563eb\'" onmouseout="this.style.color=\'#6b7280\'">π</button>'
|
| 138 |
)
|
| 139 |
-
|
| 140 |
items.append(f"""
|
| 141 |
<div style="display:flex;justify-content:flex-start;margin:14px 0 2px;">
|
| 142 |
<div style="max-width:82%;">
|
|
@@ -144,74 +139,66 @@ def render_chat_html(history, lang_code):
|
|
| 144 |
SurgiSight AI
|
| 145 |
</div>
|
| 146 |
<div style="display:flex;align-items:flex-start;gap:10px;">
|
| 147 |
-
<div style="font-size:0.9rem;line-height:1.7;flex:1;">
|
| 148 |
-
{text_html}
|
| 149 |
-
</div>
|
| 150 |
{spk_btn}
|
| 151 |
</div>
|
| 152 |
{audio_html}
|
| 153 |
</div>
|
| 154 |
</div>""")
|
| 155 |
-
|
| 156 |
scroll_js = "<script>setTimeout(function(){var e=document.getElementById('ce');if(e)e.scrollIntoView({behavior:'smooth'});},80);</script>"
|
| 157 |
return f"""
|
| 158 |
<div id="chat-scroll" style="height:440px;overflow-y:auto;padding:8px 12px;border-radius:8px;border:1px solid var(--border-color-primary,#333);">
|
| 159 |
{''.join(items)}
|
| 160 |
<div id="ce"></div>
|
| 161 |
</div>
|
| 162 |
-
{scroll_js}
|
| 163 |
-
"""
|
| 164 |
|
| 165 |
def retranslate_history(language):
|
| 166 |
-
"""Retranslate all assistant messages to new language, rebuild display."""
|
| 167 |
global _chat_history
|
| 168 |
if not _chat_history:
|
| 169 |
-
return render_chat_html([], LANGUAGES.get(language,LANGUAGES["English"])["code"])
|
| 170 |
lang_cfg = LANGUAGES.get(language, LANGUAGES["English"])
|
| 171 |
for msg in _chat_history:
|
| 172 |
if msg["role"] == "assistant":
|
| 173 |
-
if lang_cfg["code"] == "en"
|
| 174 |
-
msg["display"] = msg["en"] # restore original
|
| 175 |
-
else:
|
| 176 |
-
msg["display"] = translate_to(msg["en"], lang_cfg)
|
| 177 |
return render_chat_html(_chat_history, lang_cfg["code"])
|
| 178 |
|
| 179 |
|
|
|
|
| 180 |
def generate_pdf(original_image, annotated_image, seen, alert, explanation):
|
| 181 |
pdf_path = "/tmp/surgisight_report.pdf"
|
| 182 |
doc = SimpleDocTemplate(pdf_path, pagesize=A4,
|
| 183 |
rightMargin=2*cm, leftMargin=2*cm, topMargin=2*cm, bottomMargin=2*cm)
|
| 184 |
styles = getSampleStyleSheet()
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
story = []
|
| 194 |
-
|
| 195 |
-
story += [Spacer(1,.3*cm), Paragraph("SurgiSight",
|
| 196 |
-
Paragraph("Surgical Anatomy Analysis Report",
|
| 197 |
-
Paragraph(f"Generated on {
|
| 198 |
Spacer(1,.2*cm), HRFlowable(width="100%",thickness=2,color=colors.HexColor('#1a3a5c')),
|
| 199 |
-
Spacer(1,.4*cm), Paragraph("Segmentation Output",
|
| 200 |
-
iw,ih = 8.5*cm,
|
| 201 |
ob=io.BytesIO(); original_image.save(ob,format="PNG"); ob.seek(0)
|
| 202 |
ab=io.BytesIO(); annotated_image.save(ab,format="PNG"); ab.seek(0)
|
| 203 |
it=Table([[RLImage(ob,width=iw,height=ih),RLImage(ab,width=iw,height=ih)],
|
| 204 |
-
[Paragraph("Original Frame",
|
| 205 |
colWidths=[iw+.5*cm,iw+.5*cm])
|
| 206 |
it.setStyle(TableStyle([('ALIGN',(0,0),(-1,-1),'CENTER'),('VALIGN',(0,0),(-1,-1),'MIDDLE'),
|
| 207 |
('BACKGROUND',(0,0),(-1,0),colors.HexColor('#f5f5f5')),
|
| 208 |
('BOX',(0,0),(0,0),.5,colors.HexColor('#ddd')),('BOX',(1,0),(1,0),.5,colors.HexColor('#ddd'))]))
|
| 209 |
-
story+=[it,Spacer(1,.5*cm),Paragraph("Safety Assessment",
|
| 210 |
-
if any(d in alert for d in DANGER_CLASSES): story.append(Paragraph(f"WARNING: {alert}",
|
| 211 |
-
else: story.append(Paragraph(f"SAFE: {alert}",
|
| 212 |
-
story+=[Spacer(1,.4*cm),Paragraph("Detected Tissues & Instruments",
|
| 213 |
-
td=[["Structure","Confidence","Risk Level"]]
|
| 214 |
-
rd=[]
|
| 215 |
for name,conf in sorted(seen.items(),key=lambda x:-x[1]):
|
| 216 |
if name=="Black Background": continue
|
| 217 |
bar="\u2588"*int(conf*10)+"\u2591"*(10-int(conf*10))
|
|
@@ -230,8 +217,8 @@ def generate_pdf(original_image, annotated_image, seen, alert, explanation):
|
|
| 230 |
if isd: ds.append(('FONTNAME',(2,r),(2,r),'Helvetica-Bold'))
|
| 231 |
dt.setStyle(TableStyle(ds))
|
| 232 |
story+=[dt,Spacer(1,.5*cm),HRFlowable(width="100%",thickness=.5,color=colors.HexColor('#ccc')),
|
| 233 |
-
Spacer(1,.3*cm),Paragraph("Anatomy Teaching Note",
|
| 234 |
-
Paragraph(explanation,
|
| 235 |
HRFlowable(width="100%",thickness=.5,color=colors.HexColor('#ccc')),Spacer(1,.3*cm)]
|
| 236 |
md=[["Detection Model","YOLOv8-seg fine-tuned on CholecSeg8k (MICCAI 2020)"],
|
| 237 |
["LLM","Meta Llama 3.1 8B Instruct"],["Inference","Modal GPU (T4)"],
|
|
@@ -243,18 +230,115 @@ def generate_pdf(original_image, annotated_image, seen, alert, explanation):
|
|
| 243 |
('ROWBACKGROUNDS',(0,0),(-1,-1),[colors.HexColor('#f5f5f5'),colors.white])]))
|
| 244 |
story+=[mt,Spacer(1,.4*cm),HRFlowable(width="100%",thickness=1,color=colors.HexColor('#1a3a5c')),
|
| 245 |
Spacer(1,.2*cm),
|
| 246 |
-
Paragraph("DISCLAIMER: Research prototype only. Not a medical device. No real patient data.",
|
| 247 |
-
Paragraph("Built for Build Small Hackathon 2026",
|
| 248 |
doc.build(story)
|
| 249 |
return pdf_path
|
| 250 |
|
| 251 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
def segment_image(input_image, conf_threshold=0.25):
|
| 253 |
global last_result, chat_context, _chat_history
|
| 254 |
_chat_history = []
|
| 255 |
if input_image is None:
|
| 256 |
return (None,"Upload a frame to begin analysis.","ββ","ββ",
|
| 257 |
-
gr.update(visible=False),gr.update(visible=False),
|
| 258 |
gr.update(visible=False),gr.update(visible=False),gr.update(visible=False),
|
| 259 |
render_chat_html([],"en"))
|
| 260 |
detector = get_detector()
|
|
@@ -298,8 +382,11 @@ def segment_image(input_image, conf_threshold=0.25):
|
|
| 298 |
q2=suggested[1] if len(suggested)>1 else ""
|
| 299 |
q3=suggested[2] if len(suggested)>2 else ""
|
| 300 |
return (annotated_image, summary, alert, explanation,
|
| 301 |
-
|
| 302 |
-
|
|
|
|
|
|
|
|
|
|
| 303 |
|
| 304 |
|
| 305 |
def send_message(message, language):
|
|
@@ -312,12 +399,9 @@ def send_message(message, language):
|
|
| 312 |
system_prompt=(
|
| 313 |
"You are SurgiSight, an expert surgical anatomy assistant for medical trainees. "
|
| 314 |
+(f"Current frame detected: {', '.join(tissue_list)}. Safety status: {alert}. " if tissue_list else "")
|
| 315 |
-
+"Answer concisely in 2-4 sentences. "
|
| 316 |
-
+lang_cfg["prompt"])
|
| 317 |
-
# Build context from english originals for coherent conversation
|
| 318 |
msgs=[{"role":"system","content":system_prompt}]
|
| 319 |
-
for m in _chat_history:
|
| 320 |
-
msgs.append({"role":m["role"],"content":m["en"]})
|
| 321 |
msgs.append({"role":"user","content":message})
|
| 322 |
try:
|
| 323 |
resp=client.chat_completion(msgs,max_tokens=200,temperature=0.5)
|
|
@@ -334,6 +418,11 @@ def export_pdf():
|
|
| 334 |
return generate_pdf(last_result["original"],last_result["annotated"],
|
| 335 |
last_result["seen"],last_result["alert"],last_result["explanation"])
|
| 336 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
|
| 338 |
css = """
|
| 339 |
.gradio-container { max-width:1120px !important; margin:0 auto !important; }
|
|
@@ -365,10 +454,10 @@ overlay_js = """
|
|
| 365 |
with gr.Blocks(title="SurgiSight β Surgical AI", js=overlay_js) as demo:
|
| 366 |
|
| 367 |
gr.Markdown("""
|
| 368 |
-
#
|
| 369 |
-
**AI-powered anatomy detection for laparoscopic training** Β·
|
| 370 |
|
| 371 |
-
>
|
| 372 |
""")
|
| 373 |
|
| 374 |
with gr.Row(equal_height=True):
|
|
@@ -385,26 +474,26 @@ with gr.Blocks(title="SurgiSight β Surgical AI", js=overlay_js) as demo:
|
|
| 385 |
|
| 386 |
explain_box = gr.Textbox(label="π Anatomy Brief", lines=4, placeholder="AI explanation will appear after analysisβ¦")
|
| 387 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
with gr.Row():
|
| 389 |
-
|
| 390 |
-
|
| 391 |
|
| 392 |
with gr.Column(visible=False) as chat_section:
|
| 393 |
gr.Markdown("---\n### π¬ AI Anatomy Consult")
|
| 394 |
-
|
| 395 |
with gr.Row():
|
| 396 |
lang_select = gr.Dropdown(
|
| 397 |
choices=list(LANGUAGES.keys()), value="English",
|
| 398 |
label="π Language β switching retranslates all messages",
|
| 399 |
-
scale=0, min_width=
|
| 400 |
-
|
| 401 |
with gr.Row():
|
| 402 |
sq1 = gr.Button("", visible=False, size="sm")
|
| 403 |
sq2 = gr.Button("", visible=False, size="sm")
|
| 404 |
sq3 = gr.Button("", visible=False, size="sm")
|
| 405 |
-
|
| 406 |
chat_display = gr.HTML(value="", label="")
|
| 407 |
-
|
| 408 |
with gr.Row():
|
| 409 |
chat_input = gr.Textbox(
|
| 410 |
placeholder="Ask about anatomy, safety, or techniqueβ¦",
|
|
@@ -418,22 +507,19 @@ with gr.Blocks(title="SurgiSight β Surgical AI", js=overlay_js) as demo:
|
|
| 418 |
|
| 419 |
gr.Markdown("---\n*CholecSeg8k Β· MICCAI 2020 Β· No patient data Β· Built for Build Small Hackathon 2026*")
|
| 420 |
|
| 421 |
-
# Wiring
|
| 422 |
run_btn.click(fn=segment_image, inputs=[input_img, conf_slider],
|
| 423 |
outputs=[output_img, output_text, danger_box, explain_box,
|
| 424 |
-
pdf_btn, chat_section, sq1, sq2, sq3, chat_display])
|
| 425 |
|
| 426 |
pdf_btn.click(fn=export_pdf, outputs=[pdf_output]).then(
|
| 427 |
fn=lambda: gr.update(visible=True), outputs=[pdf_output])
|
|
|
|
|
|
|
| 428 |
|
| 429 |
-
send_btn.click(fn=send_message, inputs=[chat_input, lang_select],
|
| 430 |
-
|
| 431 |
-
chat_input.submit(fn=send_message, inputs=[chat_input, lang_select],
|
| 432 |
-
outputs=[chat_display, chat_input])
|
| 433 |
-
|
| 434 |
-
# Language change β retranslate entire chat
|
| 435 |
lang_select.change(fn=retranslate_history, inputs=[lang_select], outputs=[chat_display])
|
| 436 |
-
|
| 437 |
sq1.click(fn=send_message, inputs=[sq1, lang_select], outputs=[chat_display, chat_input])
|
| 438 |
sq2.click(fn=send_message, inputs=[sq2, lang_select], outputs=[chat_display, chat_input])
|
| 439 |
sq3.click(fn=send_message, inputs=[sq3, lang_select], outputs=[chat_display, chat_input])
|
|
|
|
| 15 |
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image as RLImage, Table, TableStyle, HRFlowable
|
| 16 |
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
|
| 17 |
from reportlab.lib.enums import TA_CENTER
|
| 18 |
+
from docx import Document
|
| 19 |
+
from docx.shared import Inches, Pt, RGBColor
|
| 20 |
+
from docx.enum.text import WD_ALIGN_PARAGRAPH
|
| 21 |
+
from docx.oxml.ns import qn
|
| 22 |
+
from docx.oxml import OxmlElement
|
| 23 |
|
| 24 |
print("All imports OK")
|
| 25 |
|
|
|
|
| 38 |
]
|
| 39 |
DANGER_CLASSES = ["Hepatic Vein","Cystic Duct","Blood"]
|
| 40 |
|
|
|
|
| 41 |
LANGUAGES = {
|
| 42 |
"English": {"code": "en", "prompt": "Respond in English."},
|
| 43 |
"French": {"code": "fr", "prompt": "RΓ©ponds en franΓ§ais."},
|
|
|
|
| 45 |
|
| 46 |
last_result = {}
|
| 47 |
chat_context = {}
|
|
|
|
|
|
|
| 48 |
_chat_history = []
|
| 49 |
|
| 50 |
try:
|
|
|
|
| 73 |
print(f"TTS error: {e}"); return ""
|
| 74 |
|
| 75 |
def translate_to(text, lang_cfg):
|
| 76 |
+
if lang_cfg["code"] == "en" or not client: return text
|
|
|
|
|
|
|
| 77 |
try:
|
| 78 |
+
lang_name = [k for k,v in LANGUAGES.items() if v["code"]==lang_cfg["code"]][0]
|
| 79 |
+
prompt = f"Translate the following medical text to {lang_name}. Output ONLY the translation, nothing else.\n\n{text}"
|
| 80 |
resp = client.chat_completion([{"role":"user","content":prompt}], max_tokens=300, temperature=0.1)
|
| 81 |
return resp.choices[0].message.content.strip()
|
| 82 |
except:
|
|
|
|
| 96 |
return questions[:3]
|
| 97 |
|
| 98 |
def render_chat_html(history, lang_code):
|
|
|
|
| 99 |
if not history:
|
| 100 |
return '<p style="color:#888;text-align:center;padding:32px 0;font-size:0.9rem;">Run analysis first, then ask questions here.</p>'
|
|
|
|
| 101 |
items = []
|
| 102 |
for i, msg in enumerate(history):
|
| 103 |
role = msg["role"]
|
| 104 |
text = msg["display"]
|
|
|
|
| 105 |
text_html = re.sub(r'\*\*(.+?)\*\*', r'<strong>\1</strong>', text)
|
| 106 |
text_html = text_html.replace("\n\n", "</p><p style='margin:6px 0;'>").replace("\n", "<br>")
|
| 107 |
text_html = f"<p style='margin:0;'>{text_html}</p>"
|
|
|
|
| 108 |
if role == "user":
|
| 109 |
items.append(f"""
|
| 110 |
<div style="display:flex;justify-content:flex-end;margin:14px 0 2px;">
|
|
|
|
| 132 |
f'border-radius:6px;color:#6b7280;flex-shrink:0;margin-top:1px;transition:color 0.15s;" '
|
| 133 |
f'onmouseover="this.style.color=\'#2563eb\'" onmouseout="this.style.color=\'#6b7280\'">π</button>'
|
| 134 |
)
|
|
|
|
| 135 |
items.append(f"""
|
| 136 |
<div style="display:flex;justify-content:flex-start;margin:14px 0 2px;">
|
| 137 |
<div style="max-width:82%;">
|
|
|
|
| 139 |
SurgiSight AI
|
| 140 |
</div>
|
| 141 |
<div style="display:flex;align-items:flex-start;gap:10px;">
|
| 142 |
+
<div style="font-size:0.9rem;line-height:1.7;flex:1;">{text_html}</div>
|
|
|
|
|
|
|
| 143 |
{spk_btn}
|
| 144 |
</div>
|
| 145 |
{audio_html}
|
| 146 |
</div>
|
| 147 |
</div>""")
|
|
|
|
| 148 |
scroll_js = "<script>setTimeout(function(){var e=document.getElementById('ce');if(e)e.scrollIntoView({behavior:'smooth'});},80);</script>"
|
| 149 |
return f"""
|
| 150 |
<div id="chat-scroll" style="height:440px;overflow-y:auto;padding:8px 12px;border-radius:8px;border:1px solid var(--border-color-primary,#333);">
|
| 151 |
{''.join(items)}
|
| 152 |
<div id="ce"></div>
|
| 153 |
</div>
|
| 154 |
+
{scroll_js}"""
|
|
|
|
| 155 |
|
| 156 |
def retranslate_history(language):
|
|
|
|
| 157 |
global _chat_history
|
| 158 |
if not _chat_history:
|
| 159 |
+
return render_chat_html([], LANGUAGES.get(language, LANGUAGES["English"])["code"])
|
| 160 |
lang_cfg = LANGUAGES.get(language, LANGUAGES["English"])
|
| 161 |
for msg in _chat_history:
|
| 162 |
if msg["role"] == "assistant":
|
| 163 |
+
msg["display"] = msg["en"] if lang_cfg["code"] == "en" else translate_to(msg["en"], lang_cfg)
|
|
|
|
|
|
|
|
|
|
| 164 |
return render_chat_html(_chat_history, lang_cfg["code"])
|
| 165 |
|
| 166 |
|
| 167 |
+
# ββ PDF βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 168 |
def generate_pdf(original_image, annotated_image, seen, alert, explanation):
|
| 169 |
pdf_path = "/tmp/surgisight_report.pdf"
|
| 170 |
doc = SimpleDocTemplate(pdf_path, pagesize=A4,
|
| 171 |
rightMargin=2*cm, leftMargin=2*cm, topMargin=2*cm, bottomMargin=2*cm)
|
| 172 |
styles = getSampleStyleSheet()
|
| 173 |
+
title_s = ParagraphStyle('T2', parent=styles['Title'], fontSize=22, textColor=colors.HexColor('#1a3a5c'), spaceAfter=4, fontName='Helvetica-Bold', alignment=TA_CENTER)
|
| 174 |
+
sub_s = ParagraphStyle('S2', parent=styles['Normal'], fontSize=10, textColor=colors.HexColor('#888'), spaceAfter=2, alignment=TA_CENTER)
|
| 175 |
+
sec_s = ParagraphStyle('Se2',parent=styles['Normal'], fontSize=13, textColor=colors.HexColor('#1a3a5c'), spaceAfter=6, spaceBefore=12, fontName='Helvetica-Bold')
|
| 176 |
+
body_s = ParagraphStyle('B2', parent=styles['Normal'], fontSize=10, textColor=colors.HexColor('#1a1a1a'), spaceAfter=4, leading=16)
|
| 177 |
+
danger_s = ParagraphStyle('D2', parent=styles['Normal'], fontSize=10, textColor=colors.HexColor('#cc0000'), spaceAfter=4, leading=16, fontName='Helvetica-Bold', backColor=colors.HexColor('#fff0f0'), borderPadding=(6,8,6,8))
|
| 178 |
+
safe_s = ParagraphStyle('Sa2',parent=styles['Normal'], fontSize=10, textColor=colors.HexColor('#1a7a40'), spaceAfter=4, leading=16, fontName='Helvetica-Bold', backColor=colors.HexColor('#f0fff4'), borderPadding=(6,8,6,8))
|
| 179 |
+
cap_s = ParagraphStyle('C2', parent=styles['Normal'], fontSize=8, textColor=colors.HexColor('#888'), alignment=TA_CENTER, spaceAfter=4)
|
| 180 |
+
foot_s = ParagraphStyle('F2', parent=styles['Normal'], fontSize=8, textColor=colors.HexColor('#aaa'), alignment=TA_CENTER)
|
| 181 |
story = []
|
| 182 |
+
ts = datetime.datetime.now().strftime("%B %d, %Y at %H:%M")
|
| 183 |
+
story += [Spacer(1,.3*cm), Paragraph("SurgiSight",title_s),
|
| 184 |
+
Paragraph("Surgical Anatomy Analysis Report",sub_s),
|
| 185 |
+
Paragraph(f"Generated on {ts}",sub_s),
|
| 186 |
Spacer(1,.2*cm), HRFlowable(width="100%",thickness=2,color=colors.HexColor('#1a3a5c')),
|
| 187 |
+
Spacer(1,.4*cm), Paragraph("Segmentation Output",sec_s)]
|
| 188 |
+
iw,ih = 8.5*cm,6.5*cm
|
| 189 |
ob=io.BytesIO(); original_image.save(ob,format="PNG"); ob.seek(0)
|
| 190 |
ab=io.BytesIO(); annotated_image.save(ab,format="PNG"); ab.seek(0)
|
| 191 |
it=Table([[RLImage(ob,width=iw,height=ih),RLImage(ab,width=iw,height=ih)],
|
| 192 |
+
[Paragraph("Original Frame",cap_s),Paragraph("AI Segmented Output",cap_s)]],
|
| 193 |
colWidths=[iw+.5*cm,iw+.5*cm])
|
| 194 |
it.setStyle(TableStyle([('ALIGN',(0,0),(-1,-1),'CENTER'),('VALIGN',(0,0),(-1,-1),'MIDDLE'),
|
| 195 |
('BACKGROUND',(0,0),(-1,0),colors.HexColor('#f5f5f5')),
|
| 196 |
('BOX',(0,0),(0,0),.5,colors.HexColor('#ddd')),('BOX',(1,0),(1,0),.5,colors.HexColor('#ddd'))]))
|
| 197 |
+
story+=[it,Spacer(1,.5*cm),Paragraph("Safety Assessment",sec_s)]
|
| 198 |
+
if any(d in alert for d in DANGER_CLASSES): story.append(Paragraph(f"WARNING: {alert}",danger_s))
|
| 199 |
+
else: story.append(Paragraph(f"SAFE: {alert}",safe_s))
|
| 200 |
+
story+=[Spacer(1,.4*cm),Paragraph("Detected Tissues & Instruments",sec_s)]
|
| 201 |
+
td=[["Structure","Confidence","Risk Level"]]; rd=[]
|
|
|
|
| 202 |
for name,conf in sorted(seen.items(),key=lambda x:-x[1]):
|
| 203 |
if name=="Black Background": continue
|
| 204 |
bar="\u2588"*int(conf*10)+"\u2591"*(10-int(conf*10))
|
|
|
|
| 217 |
if isd: ds.append(('FONTNAME',(2,r),(2,r),'Helvetica-Bold'))
|
| 218 |
dt.setStyle(TableStyle(ds))
|
| 219 |
story+=[dt,Spacer(1,.5*cm),HRFlowable(width="100%",thickness=.5,color=colors.HexColor('#ccc')),
|
| 220 |
+
Spacer(1,.3*cm),Paragraph("Anatomy Teaching Note",sec_s),
|
| 221 |
+
Paragraph(explanation,body_s),Spacer(1,.5*cm),
|
| 222 |
HRFlowable(width="100%",thickness=.5,color=colors.HexColor('#ccc')),Spacer(1,.3*cm)]
|
| 223 |
md=[["Detection Model","YOLOv8-seg fine-tuned on CholecSeg8k (MICCAI 2020)"],
|
| 224 |
["LLM","Meta Llama 3.1 8B Instruct"],["Inference","Modal GPU (T4)"],
|
|
|
|
| 230 |
('ROWBACKGROUNDS',(0,0),(-1,-1),[colors.HexColor('#f5f5f5'),colors.white])]))
|
| 231 |
story+=[mt,Spacer(1,.4*cm),HRFlowable(width="100%",thickness=1,color=colors.HexColor('#1a3a5c')),
|
| 232 |
Spacer(1,.2*cm),
|
| 233 |
+
Paragraph("DISCLAIMER: Research prototype only. Not a medical device. No real patient data.",foot_s),
|
| 234 |
+
Paragraph("Built for Build Small Hackathon 2026",foot_s)]
|
| 235 |
doc.build(story)
|
| 236 |
return pdf_path
|
| 237 |
|
| 238 |
|
| 239 |
+
# ββ Word ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 240 |
+
def generate_word(original_image, annotated_image, seen, alert, explanation):
|
| 241 |
+
docx_path = "/tmp/surgisight_report.docx"
|
| 242 |
+
doc = Document()
|
| 243 |
+
for section in doc.sections:
|
| 244 |
+
section.top_margin = Inches(1); section.bottom_margin = Inches(1)
|
| 245 |
+
section.left_margin = Inches(1.1); section.right_margin = Inches(1.1)
|
| 246 |
+
|
| 247 |
+
t = doc.add_heading("SurgiSight", 0)
|
| 248 |
+
t.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
| 249 |
+
for run in t.runs:
|
| 250 |
+
run.font.color.rgb = RGBColor(0x1a,0x3a,0x5c)
|
| 251 |
+
|
| 252 |
+
sub = doc.add_paragraph("Surgical Anatomy Analysis Report")
|
| 253 |
+
sub.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
| 254 |
+
sub.runs[0].font.size = Pt(11)
|
| 255 |
+
sub.runs[0].font.color.rgb = RGBColor(0x88,0x88,0x88)
|
| 256 |
+
|
| 257 |
+
ts = doc.add_paragraph(datetime.datetime.now().strftime("Generated on %B %d, %Y at %H:%M"))
|
| 258 |
+
ts.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
| 259 |
+
ts.runs[0].font.size = Pt(9)
|
| 260 |
+
ts.runs[0].font.color.rgb = RGBColor(0x88,0x88,0x88)
|
| 261 |
+
doc.add_paragraph()
|
| 262 |
+
|
| 263 |
+
doc.add_heading("Segmentation Output", 2)
|
| 264 |
+
img_tbl = doc.add_table(rows=2, cols=2)
|
| 265 |
+
img_tbl.style = "Table Grid"
|
| 266 |
+
for col_idx, (pil_img, caption) in enumerate([(original_image,"Original Frame"),(annotated_image,"AI Segmented Output")]):
|
| 267 |
+
buf = io.BytesIO(); pil_img.save(buf,format="PNG"); buf.seek(0)
|
| 268 |
+
cell = img_tbl.cell(0,col_idx)
|
| 269 |
+
cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.CENTER
|
| 270 |
+
cell.paragraphs[0].add_run().add_picture(buf, width=Inches(2.9))
|
| 271 |
+
cap_cell = img_tbl.cell(1,col_idx)
|
| 272 |
+
cap_cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.CENTER
|
| 273 |
+
cr = cap_cell.paragraphs[0].add_run(caption)
|
| 274 |
+
cr.font.size = Pt(9); cr.font.color.rgb = RGBColor(0x88,0x88,0x88)
|
| 275 |
+
doc.add_paragraph()
|
| 276 |
+
|
| 277 |
+
doc.add_heading("Safety Assessment", 2)
|
| 278 |
+
is_danger = any(d in alert for d in DANGER_CLASSES)
|
| 279 |
+
p = doc.add_paragraph()
|
| 280 |
+
run = p.add_run(("β WARNING: " if is_danger else "β SAFE: ") + alert)
|
| 281 |
+
run.bold = True
|
| 282 |
+
run.font.size = Pt(11)
|
| 283 |
+
run.font.color.rgb = RGBColor(0xcc,0,0) if is_danger else RGBColor(0x1a,0x7a,0x40)
|
| 284 |
+
doc.add_paragraph()
|
| 285 |
+
|
| 286 |
+
doc.add_heading("Detected Tissues & Instruments", 2)
|
| 287 |
+
rows = [(n,c) for n,c in sorted(seen.items(),key=lambda x:-x[1]) if n!="Black Background"]
|
| 288 |
+
if rows:
|
| 289 |
+
tbl = doc.add_table(rows=1+len(rows), cols=3)
|
| 290 |
+
tbl.style = "Table Grid"
|
| 291 |
+
hdr = tbl.rows[0].cells
|
| 292 |
+
for i,h in enumerate(["Structure","Confidence","Risk Level"]):
|
| 293 |
+
hdr[i].text = h
|
| 294 |
+
hdr[i].paragraphs[0].runs[0].bold = True
|
| 295 |
+
hdr[i].paragraphs[0].runs[0].font.size = Pt(10)
|
| 296 |
+
hdr[i].paragraphs[0].runs[0].font.color.rgb = RGBColor(0xff,0xff,0xff)
|
| 297 |
+
tc = hdr[i]._tc; tcPr = tc.get_or_add_tcPr()
|
| 298 |
+
shd = OxmlElement('w:shd')
|
| 299 |
+
shd.set(qn('w:val'),'clear'); shd.set(qn('w:color'),'auto'); shd.set(qn('w:fill'),'1a3a5c')
|
| 300 |
+
tcPr.append(shd)
|
| 301 |
+
for r_idx,(name,conf) in enumerate(rows):
|
| 302 |
+
row = tbl.rows[r_idx+1].cells
|
| 303 |
+
row[0].text = name
|
| 304 |
+
row[1].text = f"{conf:.1%}"
|
| 305 |
+
is_d = name in DANGER_CLASSES
|
| 306 |
+
rr = row[2].paragraphs[0].add_run("DANGER" if is_d else "Safe")
|
| 307 |
+
rr.bold = is_d
|
| 308 |
+
rr.font.size = Pt(9)
|
| 309 |
+
rr.font.color.rgb = RGBColor(0xcc,0,0) if is_d else RGBColor(0x1a,0x7a,0x40)
|
| 310 |
+
for c in [row[0],row[1]]:
|
| 311 |
+
if c.paragraphs[0].runs:
|
| 312 |
+
c.paragraphs[0].runs[0].font.size = Pt(9)
|
| 313 |
+
doc.add_paragraph()
|
| 314 |
+
|
| 315 |
+
doc.add_heading("Anatomy Teaching Note", 2)
|
| 316 |
+
doc.add_paragraph(explanation)
|
| 317 |
+
doc.add_paragraph()
|
| 318 |
+
|
| 319 |
+
doc.add_heading("Model Information", 2)
|
| 320 |
+
for label,value in [("Detection Model","YOLOv8-seg fine-tuned on CholecSeg8k (MICCAI 2020)"),
|
| 321 |
+
("LLM","Meta Llama 3.1 8B Instruct"),("Inference","Modal GPU (T4)"),
|
| 322 |
+
("Dataset","CholecSeg8k β 8,080 frames, 13 classes"),("mAP50","0.581")]:
|
| 323 |
+
p = doc.add_paragraph()
|
| 324 |
+
rl = p.add_run(f"{label}: "); rl.bold=True; rl.font.size=Pt(9); rl.font.color.rgb=RGBColor(0x1a,0x3a,0x5c)
|
| 325 |
+
rv = p.add_run(value); rv.font.size=Pt(9)
|
| 326 |
+
doc.add_paragraph()
|
| 327 |
+
|
| 328 |
+
disc = doc.add_paragraph("DISCLAIMER: Research prototype only. Not a medical device. No real patient data. Built for Build Small Hackathon 2026.")
|
| 329 |
+
disc.runs[0].font.size = Pt(8)
|
| 330 |
+
disc.runs[0].font.color.rgb = RGBColor(0xaa,0xaa,0xaa)
|
| 331 |
+
doc.save(docx_path)
|
| 332 |
+
return docx_path
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
# ββ Segment βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 336 |
def segment_image(input_image, conf_threshold=0.25):
|
| 337 |
global last_result, chat_context, _chat_history
|
| 338 |
_chat_history = []
|
| 339 |
if input_image is None:
|
| 340 |
return (None,"Upload a frame to begin analysis.","ββ","ββ",
|
| 341 |
+
gr.update(visible=False),gr.update(visible=False),gr.update(visible=False),
|
| 342 |
gr.update(visible=False),gr.update(visible=False),gr.update(visible=False),
|
| 343 |
render_chat_html([],"en"))
|
| 344 |
detector = get_detector()
|
|
|
|
| 382 |
q2=suggested[1] if len(suggested)>1 else ""
|
| 383 |
q3=suggested[2] if len(suggested)>2 else ""
|
| 384 |
return (annotated_image, summary, alert, explanation,
|
| 385 |
+
gr.update(visible=True), gr.update(visible=True), gr.update(visible=True),
|
| 386 |
+
gr.update(value=q1,visible=bool(q1)),
|
| 387 |
+
gr.update(value=q2,visible=bool(q2)),
|
| 388 |
+
gr.update(value=q3,visible=bool(q3)),
|
| 389 |
+
render_chat_html(_chat_history,"en"))
|
| 390 |
|
| 391 |
|
| 392 |
def send_message(message, language):
|
|
|
|
| 399 |
system_prompt=(
|
| 400 |
"You are SurgiSight, an expert surgical anatomy assistant for medical trainees. "
|
| 401 |
+(f"Current frame detected: {', '.join(tissue_list)}. Safety status: {alert}. " if tissue_list else "")
|
| 402 |
+
+"Answer concisely in 2-4 sentences. "+lang_cfg["prompt"])
|
|
|
|
|
|
|
| 403 |
msgs=[{"role":"system","content":system_prompt}]
|
| 404 |
+
for m in _chat_history: msgs.append({"role":m["role"],"content":m["en"]})
|
|
|
|
| 405 |
msgs.append({"role":"user","content":message})
|
| 406 |
try:
|
| 407 |
resp=client.chat_completion(msgs,max_tokens=200,temperature=0.5)
|
|
|
|
| 418 |
return generate_pdf(last_result["original"],last_result["annotated"],
|
| 419 |
last_result["seen"],last_result["alert"],last_result["explanation"])
|
| 420 |
|
| 421 |
+
def export_word():
|
| 422 |
+
if not last_result: return None
|
| 423 |
+
return generate_word(last_result["original"],last_result["annotated"],
|
| 424 |
+
last_result["seen"],last_result["alert"],last_result["explanation"])
|
| 425 |
+
|
| 426 |
|
| 427 |
css = """
|
| 428 |
.gradio-container { max-width:1120px !important; margin:0 auto !important; }
|
|
|
|
| 454 |
with gr.Blocks(title="SurgiSight β Surgical AI", js=overlay_js) as demo:
|
| 455 |
|
| 456 |
gr.Markdown("""
|
| 457 |
+
# SurgiSight β Surgical Tissue Segmentation
|
| 458 |
+
**AI-powered anatomy detection for laparoscopic training** Β· YOLOv26n-seg Β· Llama 3.1 8B Β· Modal GPU Β· CholecSeg8k Β· mAP50: 0.581
|
| 459 |
|
| 460 |
+
> Research prototype β No real patient data.
|
| 461 |
""")
|
| 462 |
|
| 463 |
with gr.Row(equal_height=True):
|
|
|
|
| 474 |
|
| 475 |
explain_box = gr.Textbox(label="π Anatomy Brief", lines=4, placeholder="AI explanation will appear after analysisβ¦")
|
| 476 |
|
| 477 |
+
# ββ Export buttons ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 478 |
+
with gr.Row():
|
| 479 |
+
pdf_btn = gr.Button("β¬ Export PDF Report", visible=False, variant="secondary")
|
| 480 |
+
word_btn = gr.Button("β¬ Export Word (.docx)", visible=False, variant="secondary")
|
| 481 |
with gr.Row():
|
| 482 |
+
pdf_output = gr.File(label="π PDF Download", visible=False)
|
| 483 |
+
word_output = gr.File(label="π Word Download", visible=False)
|
| 484 |
|
| 485 |
with gr.Column(visible=False) as chat_section:
|
| 486 |
gr.Markdown("---\n### π¬ AI Anatomy Consult")
|
|
|
|
| 487 |
with gr.Row():
|
| 488 |
lang_select = gr.Dropdown(
|
| 489 |
choices=list(LANGUAGES.keys()), value="English",
|
| 490 |
label="π Language β switching retranslates all messages",
|
| 491 |
+
scale=0, min_width=280)
|
|
|
|
| 492 |
with gr.Row():
|
| 493 |
sq1 = gr.Button("", visible=False, size="sm")
|
| 494 |
sq2 = gr.Button("", visible=False, size="sm")
|
| 495 |
sq3 = gr.Button("", visible=False, size="sm")
|
|
|
|
| 496 |
chat_display = gr.HTML(value="", label="")
|
|
|
|
| 497 |
with gr.Row():
|
| 498 |
chat_input = gr.Textbox(
|
| 499 |
placeholder="Ask about anatomy, safety, or techniqueβ¦",
|
|
|
|
| 507 |
|
| 508 |
gr.Markdown("---\n*CholecSeg8k Β· MICCAI 2020 Β· No patient data Β· Built for Build Small Hackathon 2026*")
|
| 509 |
|
| 510 |
+
# ββ Wiring ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 511 |
run_btn.click(fn=segment_image, inputs=[input_img, conf_slider],
|
| 512 |
outputs=[output_img, output_text, danger_box, explain_box,
|
| 513 |
+
pdf_btn, word_btn, chat_section, sq1, sq2, sq3, chat_display])
|
| 514 |
|
| 515 |
pdf_btn.click(fn=export_pdf, outputs=[pdf_output]).then(
|
| 516 |
fn=lambda: gr.update(visible=True), outputs=[pdf_output])
|
| 517 |
+
word_btn.click(fn=export_word, outputs=[word_output]).then(
|
| 518 |
+
fn=lambda: gr.update(visible=True), outputs=[word_output])
|
| 519 |
|
| 520 |
+
send_btn.click(fn=send_message, inputs=[chat_input, lang_select], outputs=[chat_display, chat_input])
|
| 521 |
+
chat_input.submit(fn=send_message, inputs=[chat_input, lang_select], outputs=[chat_display, chat_input])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 522 |
lang_select.change(fn=retranslate_history, inputs=[lang_select], outputs=[chat_display])
|
|
|
|
| 523 |
sq1.click(fn=send_message, inputs=[sq1, lang_select], outputs=[chat_display, chat_input])
|
| 524 |
sq2.click(fn=send_message, inputs=[sq2, lang_select], outputs=[chat_display, chat_input])
|
| 525 |
sq3.click(fn=send_message, inputs=[sq3, lang_select], outputs=[chat_display, chat_input])
|