[CI] Fix a REQUIRE_HIGHER_TOLERANCE comparison bug (pytorch#114870)

Pull Request resolved: pytorch#114870
Approved by: https://github.com/jansel

benchmarks/dynamo/timm_models.py

@@ -67,7 +67,13 @@ def pip_install(package):
     "xcit_large_24_p8_224": 4,
 }
 
-REQUIRE_HIGHER_TOLERANCE = set("sebotnet33ts_256")
+REQUIRE_HIGHER_TOLERANCE = {
+    "fbnetv3_b",
+    "hrnet_w18",
+    "inception_v3",
+    "sebotnet33ts_256",
+    "selecsls42b",
+}
 
 SCALED_COMPUTE_LOSS = {
     "ese_vovnet19b_dw",
@@ -304,8 +310,8 @@ def get_tolerance_and_cosine_flag(self, is_training, current_device, name):
         cosine = self.args.cosine
         tolerance = 1e-3
         if is_training:
-            if REQUIRE_HIGHER_TOLERANCE:
-                tolerance = 2 * 1e-2
+            if name in REQUIRE_HIGHER_TOLERANCE:
+                tolerance = 4 * 1e-2
             else:
                 tolerance = 1e-2
         return tolerance, cosine