tests: add retry conformance test framework and test always idempotent operations

noxfile.py

@@ -30,6 +30,9 @@
 DEFAULT_PYTHON_VERSION = "3.8"
 SYSTEM_TEST_PYTHON_VERSIONS = ["2.7", "3.8"]
 UNIT_TEST_PYTHON_VERSIONS = ["2.7", "3.6", "3.7", "3.8", "3.9"]
+CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.8"]
+
+_DEFAULT_STORAGE_HOST = "https://storage.googleapis.com"
 
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
 
@@ -148,6 +151,37 @@ def system(session):
         session.run("py.test", "--quiet", system_test_folder_path, *session.posargs)
 
 
+@nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS)
+def conftest_retry(session):
+    """Run the retry conformance test suite."""
+    conformance_test_path = os.path.join("tests", "conformance.py")
+    conformance_test_folder_path = os.path.join("tests", "conformance")
+
+    # Environment check: Only run tests if the STORAGE_EMULATOR_HOST is set.
+    if (
+        os.environ.get("STORAGE_EMULATOR_HOST", _DEFAULT_STORAGE_HOST)
+        == _DEFAULT_STORAGE_HOST
+    ):
+        session.skip("Set STORAGE_EMULATOR_HOST to run, skipping")
+
+    conformance_test_exists = os.path.exists(conformance_test_path)
+    conformance_test_folder_exists = os.path.exists(conformance_test_folder_path)
+    # Environment check: only run tests if found.
+    if not conformance_test_exists and not conformance_test_folder_exists:
+        session.skip("Conformance tests were not found")
+
+    session.install("pytest",)
+    session.install("-e", ".")
+
+    # Run py.test against the conformance tests.
+    if conformance_test_exists:
+        session.run("py.test", "--quiet", conformance_test_path, *session.posargs)
+    if conformance_test_folder_exists:
+        session.run(
+            "py.test", "--quiet", conformance_test_folder_path, *session.posargs
+        )
+
+
 @nox.session(python=DEFAULT_PYTHON_VERSION)
 def cover(session):
     """Run the final coverage report.

tests/conformance/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import json
+import os
+
+
+def _read_local_json(json_file):
+    here = os.path.dirname(__file__)
+    json_path = os.path.abspath(os.path.join(here, json_file))
+    with io.open(json_path, "r", encoding="utf-8-sig") as fileobj:
+        return json.load(fileobj)

tests/conformance/retry_strategy_test_data.json

@@ -0,0 +1,262 @@
+{
+    "retryStrategyTests": [
+      {
+        "id": 1,
+        "description": "always idempotent",
+        "cases": [
+          {
+            "instructions": [
+              "return-503",
+              "return-503"
+            ]
+          }
+        ],
+        "methods": [
+          {
+            "name": "storage.bucket_acl.get",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.bucket_acl.list",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.buckets.delete",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.buckets.get",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.buckets.getIamPolicy",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.buckets.insert",
+            "resources": []
+          },
+          {
+            "name": "storage.buckets.list",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.buckets.lockRententionPolicy",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.buckets.testIamPermission",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.default_object_acl.get",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.default_object_acl.list",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.hmacKey.delete",
+            "resources": []
+          },
+          {
+            "name": "storage.hmacKey.get",
+            "resources": []
+          },
+          {
+            "name": "storage.hmacKey.list",
+            "resources": []
+          },
+          {
+            "name": "storage.notifications.delete",
+            "resources": [
+              "BUCKET",
+              "NOTIFICATION"
+            ]
+          },
+          {
+            "name": "storage.notifications.get",
+            "resources": [
+              "BUCKET",
+              "NOTIFICATION"
+            ]
+          },
+          {
+            "name": "storage.notifications.list",
+            "resources": [
+              "BUCKET",
+              "NOTIFICATION"
+            ]
+          },
+          {
+            "name": "storage.object_acl.get",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.object_acl.list",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.objects.get",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.objects.list",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.serviceaccount.get",
+            "resources": []
+          }
+        ],
+        "preconditionProvided": false,
+        "expectSuccess": true
+      },
+      {
+        "id": 2,
+        "description": "conditionally idempotent retries when precondition is present",
+        "cases": [
+          {
+            "instructions": [
+              "return-503",
+              "return-503"
+            ]
+          }
+        ],
+        "methods": [
+          {
+            "name": "storage.buckets.patch",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.buckets.setIamPolicy",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.buckets.update",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.hmacKey.update",
+            "resources": []
+          },
+          {
+            "name": "storage.objects.compose",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.objects.copy",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.objects.delete",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.objects.insert",
+            "resources": [
+              "BUCKET"
+            ]
+          },
+          {
+            "name": "storage.objects.patch",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.objects.rewrite",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          },
+          {
+            "name": "storage.objects.update",
+            "resources": [
+              "BUCKET",
+              "OBJECT"
+            ]
+          }
+        ],
+        "preconditionProvided": true,
+        "expectSuccess": true
+      },
+      {
+        "id": 3,
+        "description": "conditionally idempotent no retries when precondition is absent",
+        "cases": [
+          {
+            "instructions": []
+          }
+        ],
+        "methods": [],
+        "preconditionProvided": false,
+        "expectSuccess": false
+      },
+      {
+        "id": 4,
+        "description": "non idempotent",
+        "cases": [
+          {
+            "instructions": []
+          }
+        ],
+        "methods": [],
+        "preconditionProvided": false,
+        "expectSuccess": false
+      }
+    ]
+  }