diff --git a/avy.el b/avy.el
index 17caaf5..229ce81 100644
--- a/avy.el
+++ b/avy.el
@@ -424,7 +424,8 @@ LEAF is normally ((BEG . END) . WND)."
         (goto-char beg)
         (when (cl-some (lambda (o)
                          (eq (overlay-get o 'category) 'avy))
-                       (overlays-at (1+ (point))))
+                       (overlays-in (point) (min (+ (point) len)
+                                                 (line-end-position))))
           (setq str (substring str 0 1))
           (setq len 1))
         (let* ((end (if (= beg (line-end-position))