diff --git a/plugins/gpg-agent/README.md b/plugins/gpg-agent/README.md
index d66c37a5b..cfabc4c72 100644
--- a/plugins/gpg-agent/README.md
+++ b/plugins/gpg-agent/README.md
@@ -5,10 +5,40 @@ Applies some fixes for some common issues encountered with [GPG's gpg-agent](htt
 More specifically, this plugin:
 
 - Updates the `GPG_TTY` environment variable before each shell execution. 
-- Updates the `SSH_AUTH_SOCK` environment variable if `enable-ssh-support` is turned on.
+- Updates the `SSH_AUTH_SOCK` environment variable if `gpg-agent` is your SSH
+  agent. By default the plugin checks whether `enable-ssh-support` is turned
+  on. The [`ssh-support` setting](#settings) lets you say so yourself.
 
 To use it, add `gpg-agent` to the plugins array of your zshrc file:
 
 ```zsh
 plugins=(... gpg-agent)
 ```
+
+## Settings
+
+Every time a shell starts, the plugin runs `gpgconf --list-options gpg-agent` to
+find out whether `enable-ssh-support` is turned on. That check can be slow. If
+you already know the answer, state it with the `ssh-support` style and the
+plugin skips the check. Set it in your zshrc, before Oh My Zsh is sourced.
+
+If you don't use `gpg-agent` as your SSH agent:
+
+```zsh
+zstyle ':omz:plugins:gpg-agent' ssh-support no
+```
+
+The plugin then only sets `GPG_TTY`. It leaves `SSH_AUTH_SOCK` alone, and it
+doesn't update the agent's tty before each command, which only SSH support
+needs.
+
+If you do use `gpg-agent` as your SSH agent:
+
+```zsh
+zstyle ':omz:plugins:gpg-agent' ssh-support yes
+```
+
+The plugin then sets `SSH_AUTH_SOCK` without checking. `enable-ssh-support`
+still has to be set in your `gpg-agent.conf` for SSH to work.
+
+Leave the style unset to keep the automatic check.
diff --git a/plugins/gpg-agent/gpg-agent.plugin.zsh b/plugins/gpg-agent/gpg-agent.plugin.zsh
index 1f4be20b0..47595cbfc 100644
--- a/plugins/gpg-agent/gpg-agent.plugin.zsh
+++ b/plugins/gpg-agent/gpg-agent.plugin.zsh
@@ -1,5 +1,16 @@
 export GPG_TTY=$TTY
 
+# The ssh-support style says whether gpg-agent is used as the ssh agent.
+# Setting it saves asking gpgconf, which can be slow.
+zstyle -t ':omz:plugins:gpg-agent' ssh-support
+_gpg_agent_ssh_support=$? # 0 if yes, 1 if no, 2 if not set
+
+# Everything below is only needed for ssh support
+if (( _gpg_agent_ssh_support == 1 )); then
+  unset _gpg_agent_ssh_support
+  return
+fi
+
 # Fix for passphrase prompt on the correct tty
 # See https://www.gnupg.org/documentation/manuals/gnupg/Agent-Options.html#option-_002d_002denable_002dssh_002dsupport
 function _gpg-agent_update-tty_preexec {
@@ -9,9 +20,13 @@ autoload -U add-zsh-hook
 add-zsh-hook preexec _gpg-agent_update-tty_preexec
 
 # If enable-ssh-support is set, fix ssh agent integration
-if [[ $(gpgconf --list-options gpg-agent 2>/dev/null | awk -F: '$1=="enable-ssh-support" {print $10}') = 1 ]]; then
+if (( _gpg_agent_ssh_support == 0 )) \
+  || [[ $(gpgconf --list-options gpg-agent 2>/dev/null \
+    | awk -F: '$1=="enable-ssh-support" {print $10}') = 1 ]]; then
   unset SSH_AGENT_PID
   if [[ "${gnupg_SSH_AUTH_SOCK_by:-0}" -ne $$ ]]; then
     export SSH_AUTH_SOCK="$(gpgconf --list-dirs agent-ssh-socket)"
   fi
 fi
+
+unset _gpg_agent_ssh_support