diff --git a/src/video/x11/SDL_x11events.c b/src/video/x11/SDL_x11events.c
index 043db8de5..1eef65fe5 100644
--- a/src/video/x11/SDL_x11events.c
+++ b/src/video/x11/SDL_x11events.c
@@ -1131,7 +1131,20 @@ static void X11_DispatchEvent(SDL_VideoDevice *_this, XEvent *xevent)
             printf("window %p: KeymapNotify!\n", data);
 #endif
             if (SDL_GetKeyboardFocus() != NULL) {
-                X11_UpdateKeymap(_this, SDL_TRUE);
+#ifdef SDL_VIDEO_DRIVER_X11_HAS_XKBLOOKUPKEYSYM
+                if (videodata->xkb) {
+                    XkbStateRec state;
+                    X11_XkbGetUpdatedMap(videodata->display, XkbAllClientInfoMask, videodata->xkb);
+
+                    if (X11_XkbGetState(videodata->display, XkbUseCoreKbd, &state) == Success) {
+                        unsigned int group = state.group;
+                        if (group != videodata->xkb_group) {
+                            /* Only rebuild the keymap if the layout has changed. */
+                            X11_UpdateKeymap(_this, SDL_TRUE);
+                        }
+                    }
+                }
+#endif
                 X11_ReconcileKeyboardState(_this);
             }
         } else if (xevent->type == MappingNotify) {
diff --git a/src/video/x11/SDL_x11keyboard.c b/src/video/x11/SDL_x11keyboard.c
index 543c36e4a..9a5d25c9a 100644
--- a/src/video/x11/SDL_x11keyboard.c
+++ b/src/video/x11/SDL_x11keyboard.c
@@ -347,7 +347,6 @@ void X11_UpdateKeymap(SDL_VideoDevice *_this, SDL_bool send_event)
     int i;
     SDL_Scancode scancode;
     SDL_Keymap *keymap;
-    unsigned char group = 0;
 
     keymap = SDL_CreateKeymap();
 
@@ -357,7 +356,7 @@ void X11_UpdateKeymap(SDL_VideoDevice *_this, SDL_bool send_event)
         X11_XkbGetUpdatedMap(data->display, XkbAllClientInfoMask, data->xkb);
 
         if (X11_XkbGetState(data->display, XkbUseCoreKbd, &state) == Success) {
-            group = state.group;
+            data->xkb_group = state.group;
         }
     }
 #endif
@@ -372,7 +371,7 @@ void X11_UpdateKeymap(SDL_VideoDevice *_this, SDL_bool send_event)
                 continue;
             }
 
-            KeySym keysym = X11_KeyCodeToSym(_this, i, group, keymod_masks[m].xkb_mask);
+            KeySym keysym = X11_KeyCodeToSym(_this, i, data->xkb_group, keymod_masks[m].xkb_mask);
 
             /* Note: The default SDL scancode table sets this to right alt instead of AltGr/Mode, so handle it separately. */
             if (keysym != XK_ISO_Level3_Shift) {
diff --git a/src/video/x11/SDL_x11video.h b/src/video/x11/SDL_x11video.h
index 80b5c4799..f8e567c77 100644
--- a/src/video/x11/SDL_x11video.h
+++ b/src/video/x11/SDL_x11video.h
@@ -125,6 +125,7 @@ struct SDL_VideoData
     XkbDescPtr xkb;
 #endif
     int xkb_event;
+    unsigned int xkb_group;
 
     KeyCode filter_code;
     Time filter_time;