diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst
index fd1c1271a5e37..cde2a4279cf27 100644
--- a/doc/source/whatsnew/v1.0.0.rst
+++ b/doc/source/whatsnew/v1.0.0.rst
@@ -311,6 +311,7 @@ I/O
 - Bug in :func:`DataFrame.to_string` where values were truncated using display options instead of outputting the full content (:issue:`9784`)
 - Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in ISO format with ``orient="table"`` (:issue:`28130`)
 - Bug in :func:`DataFrame.to_parquet` where writing to GCS would fail with `engine='fastparquet'` if the file did not already exist (:issue:`28326`)
+- Bug in :func:`read_hdf` closing stores that it didn't open when Exceptions are raised (:issue:`28699`)
 - Bug in :meth:`DataFrame.read_json` where using ``orient="index"`` would not maintain the order (:issue:`28557`)
 - Bug in :meth:`DataFrame.to_html` where the length of the ``formatters`` argument was not verified (:issue:`28469`)
 
diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py
index 55ccd838f8a16..0db5b1b4eecfa 100644
--- a/pandas/io/pytables.py
+++ b/pandas/io/pytables.py
@@ -396,11 +396,12 @@ def read_hdf(path_or_buf, key=None, mode="r", **kwargs):
             key = candidate_only_group._v_pathname
         return store.select(key, auto_close=auto_close, **kwargs)
     except (ValueError, TypeError, KeyError):
-        # if there is an error, close the store
-        try:
-            store.close()
-        except AttributeError:
-            pass
+        if not isinstance(path_or_buf, HDFStore):
+            # if there is an error, close the store if we opened it.
+            try:
+                store.close()
+            except AttributeError:
+                pass
 
         raise
 
diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py
index 140ee5082f55d..956438f1afdf4 100644
--- a/pandas/tests/io/pytables/test_store.py
+++ b/pandas/tests/io/pytables/test_store.py
@@ -1195,8 +1195,22 @@ def test_read_missing_key_close_store(self, setup_path):
             # read with KeyError before another write
             df.to_hdf(path, "k2")
 
-    def test_append_frame_column_oriented(self, setup_path):
+    def test_read_missing_key_opened_store(self, setup_path):
+        # GH 28699
+        with ensure_clean_path(setup_path) as path:
+            df = pd.DataFrame({"a": range(2), "b": range(2)})
+            df.to_hdf(path, "k1")
+
+            store = pd.HDFStore(path, "r")
 
+            with pytest.raises(KeyError, match="'No object named k2 in the file'"):
+                pd.read_hdf(store, "k2")
+
+            # Test that the file is still open after a KeyError and that we can
+            # still read from it.
+            pd.read_hdf(store, "k1")
+
+    def test_append_frame_column_oriented(self, setup_path):
         with ensure_clean_store(setup_path) as store:
 
             # column oriented