Revert "inductor: only do the conv+bn folding for the freezing path (p…

…ytorch#109270)"

This reverts commit c7017ff.

Reverted pytorch#109270 on behalf of https://github.com/malfet due to Broke slow test, see https://hud.pytorch.org/pytorch/pytorch/commit/c7017fff38e73210541124739ed9404492ddd68c ([comment](pytorch#109270 (comment)))

torch/_inductor/fx_passes/pre_grad.py

@@ -89,11 +89,12 @@ def fuse_fx(gm: torch.fx.GraphModule, example_inputs):
         gm = permute_matmul_fusion(gm)
 
     # make sure the autograd is disabled.
-    if torch.is_grad_enabled() or not is_cpu:
+    if torch.is_grad_enabled():
         return gm
-    if config.freezing:
-        gm = remove_identity(gm)
-        gm = fuse_conv_bn(gm)
+    if not is_cpu:
+        return gm
+    gm = remove_identity(gm)
+    gm = fuse_conv_bn(gm)
     return gm