diff --git a/doc/source/whatsnew/v2.1.1.rst b/doc/source/whatsnew/v2.1.1.rst
index c9ab496295d85..0dc113bd9ab7f 100644
--- a/doc/source/whatsnew/v2.1.1.rst
+++ b/doc/source/whatsnew/v2.1.1.rst
@@ -26,6 +26,7 @@ Fixed regressions
 - Fixed regression in :meth:`Series.drop_duplicates` for PyArrow strings (:issue:`54904`)
 - Fixed regression in :meth:`Series.interpolate` raising when ``fill_value`` was given (:issue:`54920`)
 - Fixed regression in :meth:`Series.value_counts` raising for numeric data if ``bins`` was specified (:issue:`54857`)
+- Fixed regression in comparison operations for PyArrow backed columns not propagating exceptions correctly (:issue:`54944`)
 - Fixed regression when comparing a :class:`Series` with ``datetime64`` dtype with ``None`` (:issue:`54870`)
 
 .. ---------------------------------------------------------------------------
diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py
index e67b7035822cc..4b79d0dbb683e 100644
--- a/pandas/core/arrays/arrow/array.py
+++ b/pandas/core/arrays/arrow/array.py
@@ -627,20 +627,22 @@ def __setstate__(self, state) -> None:
 
     def _cmp_method(self, other, op):
         pc_func = ARROW_CMP_FUNCS[op.__name__]
-        try:
+        if isinstance(other, (ArrowExtensionArray, np.ndarray, list, BaseMaskedArray)):
             result = pc_func(self._pa_array, self._box_pa(other))
-        except (pa.lib.ArrowNotImplementedError, pa.lib.ArrowInvalid):
-            if is_scalar(other):
+        elif is_scalar(other):
+            try:
+                result = pc_func(self._pa_array, self._box_pa(other))
+            except (pa.lib.ArrowNotImplementedError, pa.lib.ArrowInvalid):
                 mask = isna(self) | isna(other)
                 valid = ~mask
                 result = np.zeros(len(self), dtype="bool")
                 result[valid] = op(np.array(self)[valid], other)
                 result = pa.array(result, type=pa.bool_())
                 result = pc.if_else(valid, result, None)
-            else:
-                raise NotImplementedError(
-                    f"{op.__name__} not implemented for {type(other)}"
-                )
+        else:
+            raise NotImplementedError(
+                f"{op.__name__} not implemented for {type(other)}"
+            )
         return ArrowExtensionArray(result)
 
     def _evaluate_op_method(self, other, op, arrow_funcs):
diff --git a/pandas/tests/extension/test_arrow.py b/pandas/tests/extension/test_arrow.py
index 8968b9a7f25fe..339e97e735f85 100644
--- a/pandas/tests/extension/test_arrow.py
+++ b/pandas/tests/extension/test_arrow.py
@@ -3027,6 +3027,14 @@ def test_duration_fillna_numpy(pa_type):
     tm.assert_series_equal(result, expected)
 
 
+def test_comparison_not_propagating_arrow_error():
+    # GH#54944
+    a = pd.Series([1 << 63], dtype="uint64[pyarrow]")
+    b = pd.Series([None], dtype="int64[pyarrow]")
+    with pytest.raises(pa.lib.ArrowInvalid, match="Integer value"):
+        a < b
+
+
 def test_factorize_chunked_dictionary():
     # GH 54844
     pa_array = pa.chunked_array(