diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 51c5a3f..0814c34 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -41,3 +41,7 @@ jobs:
     - name: Run ruff
       run: |
         ruff .
+    - name: Run test-llm-load-plugins.sh
+      run: |
+        llm install llm-cluster llm-mistral
+        ./tests/test-llm-load-plugins.sh
diff --git a/tests/test-llm-load-plugins.sh b/tests/test-llm-load-plugins.sh
new file mode 100755
index 0000000..7629798
--- /dev/null
+++ b/tests/test-llm-load-plugins.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# This should only run in environments where both
+# llm-cluster and llm-mistral are installed
+
+PLUGINS=$(llm plugins)
+echo "$PLUGINS" | jq 'any(.[]; .name == "llm-mistral")' | \
+  grep -q true || ( \
+    echo "Test failed: llm-mistral not found" && \
+    exit 1 \
+  )
+# With the LLM_LOAD_PLUGINS we should not see that
+PLUGINS2=$(LLM_LOAD_PLUGINS=llm-cluster llm plugins)
+echo "$PLUGINS2" | jq 'any(.[]; .name == "llm-mistral")' | \
+  grep -q false || ( \
+    echo "Test failed: llm-mistral should not have been loaded" && \
+    exit 1 \
+  )
+echo "$PLUGINS2" | jq 'any(.[]; .name == "llm-cluster")' | \
+  grep -q true || ( \
+    echo "Test llm-cluster should have been loaded" && \
+    exit 1 \
+  )
+# With LLM_LOAD_PLUGINS='' we should see no plugins
+PLUGINS3=$(LLM_LOAD_PLUGINS='' llm plugins)
+echo "$PLUGINS3"| \
+  grep -q '\[\]' || ( \
+    echo "Test failed: plugins should have returned []" && \
+    exit 1 \
+  )