diff --git a/llm/cli.py b/llm/cli.py
index 4656638..a62dcba 100644
--- a/llm/cli.py
+++ b/llm/cli.py
@@ -450,7 +450,13 @@ def prompt(
                 if extract:
                     text = extract_first_fenced_code_block(text) or text
                 print(text)
+    # List of exceptions that should never be raised in pytest:
+    except (ValueError, NotImplementedError) as ex:
+        raise click.ClickException(str(ex))
     except Exception as ex:
+        # All other exceptions should raise in pytest, show to user otherwise
+        if getattr(sys, "_called_from_test", False):
+            raise
         raise click.ClickException(str(ex))
 
     if isinstance(response, AsyncResponse):