diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08261befd2..ed74a364c2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,6 +38,7 @@ Ansible Changes By Release
 * Fix OpenBSD pkg_mgr fact (https://github.com/ansible/ansible/issues/30623)
 * Fix encoding error when there are nonascii values in the path to the ssh binary
 * removed YAML inventory group name validation, broke existing setups and should be global in any case, and configurable
+* performance improvment for inventory, had slown down considerably from 2.3
 
 <a id="2.4"></a>
 
diff --git a/lib/ansible/inventory/group.py b/lib/ansible/inventory/group.py
index dd90c97ea6..0933e1ab5d 100644
--- a/lib/ansible/inventory/group.py
+++ b/lib/ansible/inventory/group.py
@@ -31,6 +31,7 @@ class Group:
         self.depth = 0
         self.name = name
         self.hosts = []
+        self._hosts = set()
         self.vars = {}
         self.child_groups = []
         self.parent_groups = []
@@ -54,11 +55,14 @@ class Group:
         for parent in self.parent_groups:
             parent_groups.append(parent.serialize())
 
+        self._hosts = None
+
         result = dict(
             name=self.name,
             vars=self.vars.copy(),
             parent_groups=parent_groups,
             depth=self.depth,
+            hosts=self.hosts,
         )
 
         return result
@@ -68,6 +72,9 @@ class Group:
         self.name = data.get('name')
         self.vars = data.get('vars', dict())
         self.depth = data.get('depth', 0)
+        self.hosts = data.get('hosts', {})
+
+        self._hosts = set(self.hosts)
 
         parent_groups = data.get('parent_groups', [])
         for parent_data in parent_groups:
@@ -112,17 +119,19 @@ class Group:
             raise AnsibleError("The group named '%s' has a recursive dependency loop." % self.name)
 
     def add_host(self, host):
-        if host in self.hosts:
-            return
-        self.hosts.append(host)
-        host.add_group(self)
-        self.clear_hosts_cache()
+        if host.name not in self._hosts:
+            self.hosts.append(host)
+            self._hosts.add(host.name)
+            host.add_group(self)
+            self.clear_hosts_cache()
 
     def remove_host(self, host):
 
-        self.hosts.remove(host)
-        host.remove_group(self)
-        self.clear_hosts_cache()
+        if host.name in self._hosts:
+            self.hosts.remove(host)
+            self._hosts.remove(host.name)
+            host.remove_group(self)
+            self.clear_hosts_cache()
 
     def set_variable(self, key, value):