diff --git a/avy.el b/avy.el
index 3cffb73..26a3cbc 100644
--- a/avy.el
+++ b/avy.el
@@ -595,7 +595,7 @@ When PRED is non-nil, it's a filter for matching point positions.
 When GROUP is non-nil, (BEG . END) should delimit that regex group."
   (setq group (or group 0))
   (let ((case-fold-search (or avy-case-fold-search
-                              (not (string= regex (upcase regex)))))
+                              (string= regex (downcase regex))))
         candidates)
     (avy-dowindows current-prefix-arg
       (dolist (pair (avy--find-visible-regions