From c03d6ae7007666db35c13e941a6291c2197a537d Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Wed, 1 Jul 2020 18:31:20 +0530 Subject: [PATCH 1/8] feat(storage): add support of custom time metadata and timestamp --- google/cloud/storage/blob.py | 15 ++++++++++++ google/cloud/storage/bucket.py | 30 +++++++++++++++++++++++ tests/system/test_system.py | 16 ++++++++++-- tests/unit/test_blob.py | 19 ++++++++++++++ tests/unit/test_bucket.py | 45 ++++++++++++++++++++++++++++++++++ 5 files changed, 123 insertions(+), 2 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index ec6c6b08e..eed4fb6b5 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -3003,6 +3003,21 @@ def updated(self): if value is not None: return _rfc3339_to_datetime(value) + @property + def custom_time(self): + """Retrieve the custom time for the object. + + See https://cloud.google.com/storage/docs/json_api/v1/objects + + :rtype: :class:`datetime.datetime` or ``NoneType`` + :returns: Datetime object parsed from RFC3339 valid timestamp, or + ``None`` if the blob's resource has not been loaded from + the server (see :meth:`reload`). + """ + value = self._properties.get("customTime") + if value is not None: + return _rfc3339_to_datetime(value) + def _get_encryption_headers(key, source=False): """Builds customer encryption key headers diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index ad7eaf6df..d0b5d9860 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -170,6 +170,18 @@ class LifecycleRuleConditions(dict): :param number_of_newer_versions: (Optional) Apply rule action to versioned items having N newer versions. + :type days_since_custom_time: int + :param days_since_custom_time: (Optional) Apply rule action to items whose number of days + elapsed since the custom timestamp. This condition is relevant + only for versioned objects. The value of the field must be a non + negative integer. If it's zero, the object version will become + eligible for lifecycle action as soon as it becomes custom. + + :type custom_time_before: :class:`datetime.datetime` + :param custom_time_before: (Optional) Datetime object parsed from RFC3339 valid timestamp, apply + rule action to items whose custom time is before this timestamp. + This condition is relevant only for versioned objects. + :raises ValueError: if no arguments are passed. """ @@ -180,6 +192,8 @@ def __init__( is_live=None, matches_storage_class=None, number_of_newer_versions=None, + days_since_custom_time=None, + custom_time_before=None, _factory=False, ): conditions = {} @@ -199,6 +213,12 @@ def __init__( if number_of_newer_versions is not None: conditions["numNewerVersions"] = number_of_newer_versions + if days_since_custom_time is not None: + conditions["daysSinceCustomTime"] = days_since_custom_time + + if custom_time_before is not None: + conditions["customTimeBefore"] = _datetime_to_rfc3339(custom_time_before) + if not _factory and not conditions: raise ValueError("Supply at least one condition") @@ -245,6 +265,16 @@ def number_of_newer_versions(self): """Conditon's 'number_of_newer_versions' value.""" return self.get("numNewerVersions") + @property + def days_since_custom_time(self): + """Conditon's 'days_since_custom_time' value.""" + return self.get("daysSinceCustomTime") + + @property + def custom_time_before(self): + """Conditon's 'custom_time_before' value.""" + return self.get("customTimeBefore") + class LifecycleRuleDelete(dict): """Map a lifecycle rule deleting matching items. diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 2afc1e515..247796b88 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -191,14 +191,21 @@ def test_bucket_create_w_alt_storage_class(self): self.assertEqual(created.storage_class, constants.ARCHIVE_STORAGE_CLASS) def test_lifecycle_rules(self): + import datetime from google.cloud.storage import constants new_bucket_name = "w-lifcycle-rules" + unique_resource_id("-") + custom_time_before = datetime.datetime.now() + datetime.timedelta(days=10) self.assertRaises( exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name ) bucket = Config.CLIENT.bucket(new_bucket_name) - bucket.add_lifecycle_delete_rule(age=42) + bucket.add_lifecycle_delete_rule( + age=42, + number_of_newer_versions=3, + days_since_custom_time=2, + custom_time_before=custom_time_before, + ) bucket.add_lifecycle_set_storage_class_rule( constants.COLDLINE_STORAGE_CLASS, is_live=False, @@ -206,7 +213,12 @@ def test_lifecycle_rules(self): ) expected_rules = [ - LifecycleRuleDelete(age=42), + LifecycleRuleDelete( + age=42, + number_of_newer_versions=3, + days_since_custom_time=2, + custom_time_before=custom_time_before, + ), LifecycleRuleSetStorageClass( constants.COLDLINE_STORAGE_CLASS, is_live=False, diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 001f8801f..0ceb3cf5d 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -157,6 +157,7 @@ def _set_properties_helper(self, kms_key_name=None): "crc32c": CRC32C, "componentCount": COMPONENT_COUNT, "etag": ETAG, + "customTime": NOW, } if kms_key_name is not None: @@ -188,6 +189,7 @@ def _set_properties_helper(self, kms_key_name=None): self.assertEqual(blob.crc32c, CRC32C) self.assertEqual(blob.component_count, COMPONENT_COUNT) self.assertEqual(blob.etag, ETAG) + self.assertEqual(blob.custom_time, now) if kms_key_name is not None: self.assertEqual(blob.kms_key_name, kms_key_name) @@ -3790,6 +3792,23 @@ def test_updated_unset(self): blob = self._make_one("blob-name", bucket=BUCKET) self.assertIsNone(blob.updated) + def test_custom_time(self): + from google.cloud._helpers import _RFC3339_MICROS + from google.cloud._helpers import UTC + + BLOB_NAME = "blob-name" + bucket = _Bucket() + TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) + TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) + properties = {"customTime": TIME_CREATED} + blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) + self.assertEqual(blob.custom_time, TIMESTAMP) + + def test_custom_time_unset(self): + BUCKET = object() + blob = self._make_one("blob-name", bucket=BUCKET) + self.assertIsNone(blob.custom_time) + def test_from_string_w_valid_uri(self): from google.cloud.storage.blob import Blob diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 3c5f2e68d..24eccaa15 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -88,8 +88,47 @@ def test_ctor_w_number_of_newer_versions(self): self.assertIsNone(conditions.matches_storage_class) self.assertEqual(conditions.number_of_newer_versions, 3) + def test_ctor_w_days_since_custom_time(self): + conditions = self._make_one( + number_of_newer_versions=3, days_since_custom_time=2 + ) + expected = {"numNewerVersions": 3, "daysSinceCustomTime": 2} + self.assertEqual(dict(conditions), expected) + self.assertIsNone(conditions.age) + self.assertIsNone(conditions.created_before) + self.assertIsNone(conditions.is_live) + self.assertIsNone(conditions.matches_storage_class) + self.assertEqual(conditions.number_of_newer_versions, 3) + self.assertEqual(conditions.days_since_custom_time, 2) + + def test_ctor_w_custom_time_before(self): + import datetime + from google.cloud._helpers import _datetime_to_rfc3339 + + custom_time_before = datetime.datetime.now() + datetime.timedelta(days=10) + conditions = self._make_one( + number_of_newer_versions=3, custom_time_before=custom_time_before + ) + expected = { + "numNewerVersions": 3, + "customTimeBefore": _datetime_to_rfc3339(custom_time_before), + } + + self.assertEqual(dict(conditions), expected) + self.assertIsNone(conditions.age) + self.assertIsNone(conditions.created_before) + self.assertIsNone(conditions.is_live) + self.assertIsNone(conditions.matches_storage_class) + self.assertEqual(conditions.number_of_newer_versions, 3) + self.assertEqual( + conditions.custom_time_before, _datetime_to_rfc3339(custom_time_before) + ) + def test_from_api_repr(self): import datetime + from google.cloud._helpers import _datetime_to_rfc3339 + + custom_time_before = datetime.datetime.now() + datetime.timedelta(days=10) before = datetime.date(2018, 8, 1) klass = self._get_target_class() @@ -99,6 +138,8 @@ def test_from_api_repr(self): "isLive": True, "matchesStorageClass": ["COLDLINE"], "numNewerVersions": 3, + "daysSinceCustomTime": 2, + "customTimeBefore": _datetime_to_rfc3339(custom_time_before), } conditions = klass.from_api_repr(resource) self.assertEqual(conditions.age, 10) @@ -106,6 +147,10 @@ def test_from_api_repr(self): self.assertEqual(conditions.is_live, True) self.assertEqual(conditions.matches_storage_class, ["COLDLINE"]) self.assertEqual(conditions.number_of_newer_versions, 3) + self.assertEqual(conditions.days_since_custom_time, 2) + self.assertEqual( + conditions.custom_time_before, _datetime_to_rfc3339(custom_time_before) + ) class Test_LifecycleRuleDelete(unittest.TestCase): From 0212ca40196208c40528386869c7027f05215c38 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Fri, 17 Jul 2020 14:04:16 +0530 Subject: [PATCH 2/8] feat(storage): change the return type of custom_time_before --- google/cloud/storage/bucket.py | 4 +++- tests/unit/test_bucket.py | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index d0b5d9860..dbdd498e4 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -273,7 +273,9 @@ def days_since_custom_time(self): @property def custom_time_before(self): """Conditon's 'custom_time_before' value.""" - return self.get("customTimeBefore") + timestamp = self.get("customTimeBefore") + if timestamp is not None: + return _rfc3339_to_datetime(timestamp) class LifecycleRuleDelete(dict): diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 24eccaa15..dbebcab96 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -77,6 +77,8 @@ def test_ctor_w_created_before_and_is_live(self): self.assertEqual(conditions.is_live, False) self.assertIsNone(conditions.matches_storage_class) self.assertIsNone(conditions.number_of_newer_versions) + self.assertIsNone(conditions.days_since_custom_time) + self.assertIsNone(conditions.custom_time_before) def test_ctor_w_number_of_newer_versions(self): conditions = self._make_one(number_of_newer_versions=3) @@ -103,9 +105,12 @@ def test_ctor_w_days_since_custom_time(self): def test_ctor_w_custom_time_before(self): import datetime + import pytz from google.cloud._helpers import _datetime_to_rfc3339 - custom_time_before = datetime.datetime.now() + datetime.timedelta(days=10) + custom_time_before = datetime.datetime.utcnow().replace( + tzinfo=pytz.UTC + ) + datetime.timedelta(days=10) conditions = self._make_one( number_of_newer_versions=3, custom_time_before=custom_time_before ) @@ -120,15 +125,16 @@ def test_ctor_w_custom_time_before(self): self.assertIsNone(conditions.is_live) self.assertIsNone(conditions.matches_storage_class) self.assertEqual(conditions.number_of_newer_versions, 3) - self.assertEqual( - conditions.custom_time_before, _datetime_to_rfc3339(custom_time_before) - ) + self.assertEqual(conditions.custom_time_before, custom_time_before) def test_from_api_repr(self): import datetime + import pytz from google.cloud._helpers import _datetime_to_rfc3339 - custom_time_before = datetime.datetime.now() + datetime.timedelta(days=10) + custom_time_before = datetime.datetime.utcnow().replace( + tzinfo=pytz.UTC + ) + datetime.timedelta(days=10) before = datetime.date(2018, 8, 1) klass = self._get_target_class() @@ -148,9 +154,7 @@ def test_from_api_repr(self): self.assertEqual(conditions.matches_storage_class, ["COLDLINE"]) self.assertEqual(conditions.number_of_newer_versions, 3) self.assertEqual(conditions.days_since_custom_time, 2) - self.assertEqual( - conditions.custom_time_before, _datetime_to_rfc3339(custom_time_before) - ) + self.assertEqual(conditions.custom_time_before, custom_time_before) class Test_LifecycleRuleDelete(unittest.TestCase): From a6c830c5e56c64fcc1b9697b424a0d25785b2dec Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Fri, 24 Jul 2020 12:03:02 +0530 Subject: [PATCH 3/8] feat(storage): add setter method --- google/cloud/storage/blob.py | 13 +++++++++++++ tests/unit/test_blob.py | 13 ++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 9cf2bad09..9aca2109b 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -51,6 +51,7 @@ from google.api_core.iam import Policy from google.cloud import exceptions from google.cloud._helpers import _bytes_to_unicode +from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud._helpers import _rfc3339_to_datetime from google.cloud._helpers import _to_bytes from google.cloud.exceptions import NotFound @@ -3018,6 +3019,18 @@ def custom_time(self): if value is not None: return _rfc3339_to_datetime(value) + @custom_time.setter + def custom_time(self, value): + """Set the custom time for the object. + + See https://cloud.google.com/storage/docs/json_api/v1/objects + + :type value: :class:`datetime.datetime` + :param value: (Optional) Set the custom time of blob. Datetime object parsed + from RFC3339 valid timestamp. + """ + self._properties["customTime"] = _datetime_to_rfc3339(value) + def _get_encryption_headers(key, source=False): """Builds customer encryption key headers diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 1bac38f9a..3b2eaaa39 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -3792,7 +3792,7 @@ def test_updated_unset(self): blob = self._make_one("blob-name", bucket=BUCKET) self.assertIsNone(blob.updated) - def test_custom_time(self): + def test_custom_time_getter(self): from google.cloud._helpers import _RFC3339_MICROS from google.cloud._helpers import UTC @@ -3804,6 +3804,17 @@ def test_custom_time(self): blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) self.assertEqual(blob.custom_time, TIMESTAMP) + def test_custom_time_setter(self): + from google.cloud._helpers import UTC + + BLOB_NAME = "blob-name" + bucket = _Bucket() + TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) + blob = self._make_one(BLOB_NAME, bucket=bucket) + self.assertIsNone(blob.custom_time) + blob.custom_time = TIMESTAMP + self.assertEqual(blob.custom_time, TIMESTAMP) + def test_custom_time_unset(self): BUCKET = object() blob = self._make_one("blob-name", bucket=BUCKET) From 75e6067c9093d0668c074d904be63903bfb6fc7f Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Mon, 27 Jul 2020 16:40:29 +0530 Subject: [PATCH 4/8] feat(storage): add test for None value --- google/cloud/storage/blob.py | 7 +++++-- tests/unit/test_blob.py | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 1fd2a54de..c48a6716b 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -3196,11 +3196,14 @@ def custom_time(self, value): See https://cloud.google.com/storage/docs/json_api/v1/objects - :type value: :class:`datetime.datetime` + :type value: :class:`datetime.datetime` or ``NoneType`` :param value: (Optional) Set the custom time of blob. Datetime object parsed from RFC3339 valid timestamp. """ - self._properties["customTime"] = _datetime_to_rfc3339(value) + if value is not None: + value = _datetime_to_rfc3339(value) + + self._properties["customTime"] = value def _get_encryption_headers(key, source=False): diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 5f6c56b87..e0b3ea337 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -4152,6 +4152,14 @@ def test_custom_time_setter(self): blob.custom_time = TIMESTAMP self.assertEqual(blob.custom_time, TIMESTAMP) + def test_custom_time_setter_none_value(self): + BLOB_NAME = "blob-name" + bucket = _Bucket() + TIMESTAMP = None + blob = self._make_one(BLOB_NAME, bucket=bucket) + blob.custom_time = TIMESTAMP + self.assertIsNone(blob.custom_time) + def test_custom_time_unset(self): BUCKET = object() blob = self._make_one("blob-name", bucket=BUCKET) From 0bd9bf48c055e903ef299632e3d1313a0674819a Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Wed, 5 Aug 2020 11:26:58 +0530 Subject: [PATCH 5/8] feat(storage): changes in unittest --- tests/unit/test_blob.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 7ebcaa712..840b6bf51 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -4172,11 +4172,17 @@ def test_custom_time_setter(self): self.assertEqual(blob.custom_time, TIMESTAMP) def test_custom_time_setter_none_value(self): + from google.cloud._helpers import _RFC3339_MICROS + from google.cloud._helpers import UTC + BLOB_NAME = "blob-name" bucket = _Bucket() - TIMESTAMP = None - blob = self._make_one(BLOB_NAME, bucket=bucket) - blob.custom_time = TIMESTAMP + TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) + TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) + properties = {"customTime": TIME_CREATED} + blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) + self.assertEqual(blob.custom_time, TIMESTAMP) + blob.custom_time = None self.assertIsNone(blob.custom_time) def test_custom_time_unset(self): From fee993fed4f5182f467ceb037b06395021b86e63 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Wed, 5 Aug 2020 16:01:19 +0530 Subject: [PATCH 6/8] feat(storage): change custom_time type to date --- google/cloud/storage/blob.py | 15 +++++++------- google/cloud/storage/bucket.py | 15 +++++++------- tests/system/test_system.py | 2 +- tests/unit/test_blob.py | 36 ++++++++++++++-------------------- tests/unit/test_bucket.py | 17 ++++------------ 5 files changed, 34 insertions(+), 51 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index c48a6716b..db7f385c1 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -50,9 +50,9 @@ from google.resumable_media.requests import ResumableUpload from google.api_core.iam import Policy +from google.api_core import datetime_helpers from google.cloud import exceptions from google.cloud._helpers import _bytes_to_unicode -from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud._helpers import _rfc3339_to_datetime from google.cloud._helpers import _to_bytes from google.cloud.exceptions import NotFound @@ -3181,14 +3181,14 @@ def custom_time(self): See https://cloud.google.com/storage/docs/json_api/v1/objects - :rtype: :class:`datetime.datetime` or ``NoneType`` - :returns: Datetime object parsed from RFC3339 valid timestamp, or + :rtype: :class:`datetime.date` or ``NoneType`` + :returns: Date object parsed from iso8601 valid date, or ``None`` if the blob's resource has not been loaded from the server (see :meth:`reload`). """ value = self._properties.get("customTime") if value is not None: - return _rfc3339_to_datetime(value) + return datetime_helpers.from_iso8601_date(value) @custom_time.setter def custom_time(self, value): @@ -3196,12 +3196,11 @@ def custom_time(self, value): See https://cloud.google.com/storage/docs/json_api/v1/objects - :type value: :class:`datetime.datetime` or ``NoneType`` - :param value: (Optional) Set the custom time of blob. Datetime object parsed - from RFC3339 valid timestamp. + :type value: :class:`datetime.date` or ``NoneType`` + :param value: (Optional) Set the custom time of blob. """ if value is not None: - value = _datetime_to_rfc3339(value) + value = value.isoformat() self._properties["customTime"] = value diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index dbdd498e4..b727375f9 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -177,10 +177,9 @@ class LifecycleRuleConditions(dict): negative integer. If it's zero, the object version will become eligible for lifecycle action as soon as it becomes custom. - :type custom_time_before: :class:`datetime.datetime` - :param custom_time_before: (Optional) Datetime object parsed from RFC3339 valid timestamp, apply - rule action to items whose custom time is before this timestamp. - This condition is relevant only for versioned objects. + :type custom_time_before: :class:`datetime.date` + :param custom_time_before: (Optional) Apply rule action to items whose custom time is before this + date. This condition is relevant only for versioned objects. :raises ValueError: if no arguments are passed. """ @@ -217,7 +216,7 @@ def __init__( conditions["daysSinceCustomTime"] = days_since_custom_time if custom_time_before is not None: - conditions["customTimeBefore"] = _datetime_to_rfc3339(custom_time_before) + conditions["customTimeBefore"] = custom_time_before.isoformat() if not _factory and not conditions: raise ValueError("Supply at least one condition") @@ -273,9 +272,9 @@ def days_since_custom_time(self): @property def custom_time_before(self): """Conditon's 'custom_time_before' value.""" - timestamp = self.get("customTimeBefore") - if timestamp is not None: - return _rfc3339_to_datetime(timestamp) + before = self.get("customTimeBefore") + if before is not None: + return datetime_helpers.from_iso8601_date(before) class LifecycleRuleDelete(dict): diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 2b60d5989..8ecf8ac6b 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -196,7 +196,7 @@ def test_lifecycle_rules(self): from google.cloud.storage import constants new_bucket_name = "w-lifcycle-rules" + unique_resource_id("-") - custom_time_before = datetime.datetime.now() + datetime.timedelta(days=10) + custom_time_before = datetime.date(2018, 8, 1) self.assertRaises( exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name ) diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 840b6bf51..434a16f6a 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -134,6 +134,8 @@ def _set_properties_helper(self, kms_key_name=None): CRC32C = "FACE0DAC" COMPONENT_COUNT = 2 ETAG = "ETAG" + custom_time = datetime.date(2018, 8, 1) + CUSTOM_TIME = custom_time.isoformat() resource = { "id": BLOB_ID, "selfLink": SELF_LINK, @@ -157,7 +159,7 @@ def _set_properties_helper(self, kms_key_name=None): "crc32c": CRC32C, "componentCount": COMPONENT_COUNT, "etag": ETAG, - "customTime": NOW, + "customTime": CUSTOM_TIME, } if kms_key_name is not None: @@ -189,7 +191,7 @@ def _set_properties_helper(self, kms_key_name=None): self.assertEqual(blob.crc32c, CRC32C) self.assertEqual(blob.component_count, COMPONENT_COUNT) self.assertEqual(blob.etag, ETAG) - self.assertEqual(blob.custom_time, now) + self.assertEqual(blob.custom_time, custom_time) if kms_key_name is not None: self.assertEqual(blob.kms_key_name, kms_key_name) @@ -4149,39 +4151,31 @@ def test_updated_unset(self): self.assertIsNone(blob.updated) def test_custom_time_getter(self): - from google.cloud._helpers import _RFC3339_MICROS - from google.cloud._helpers import UTC - BLOB_NAME = "blob-name" bucket = _Bucket() - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) - TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) - properties = {"customTime": TIME_CREATED} + custom_time = datetime.date(2018, 8, 1) + CUSTOM_TIME = custom_time.isoformat() + properties = {"customTime": CUSTOM_TIME} blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.custom_time, TIMESTAMP) + self.assertEqual(blob.custom_time, custom_time) def test_custom_time_setter(self): - from google.cloud._helpers import UTC - BLOB_NAME = "blob-name" bucket = _Bucket() - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) + custom_time = datetime.date(2018, 8, 1) blob = self._make_one(BLOB_NAME, bucket=bucket) self.assertIsNone(blob.custom_time) - blob.custom_time = TIMESTAMP - self.assertEqual(blob.custom_time, TIMESTAMP) + blob.custom_time = custom_time + self.assertEqual(blob.custom_time, custom_time) def test_custom_time_setter_none_value(self): - from google.cloud._helpers import _RFC3339_MICROS - from google.cloud._helpers import UTC - BLOB_NAME = "blob-name" bucket = _Bucket() - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) - TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) - properties = {"customTime": TIME_CREATED} + custom_time = datetime.date(2018, 8, 1) + CUSTOM_TIME = custom_time.isoformat() + properties = {"customTime": CUSTOM_TIME} blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.custom_time, TIMESTAMP) + self.assertEqual(blob.custom_time, custom_time) blob.custom_time = None self.assertIsNone(blob.custom_time) diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index dbebcab96..7d8d9783f 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -105,18 +105,14 @@ def test_ctor_w_days_since_custom_time(self): def test_ctor_w_custom_time_before(self): import datetime - import pytz - from google.cloud._helpers import _datetime_to_rfc3339 - custom_time_before = datetime.datetime.utcnow().replace( - tzinfo=pytz.UTC - ) + datetime.timedelta(days=10) + custom_time_before = datetime.date(2018, 8, 1) conditions = self._make_one( number_of_newer_versions=3, custom_time_before=custom_time_before ) expected = { "numNewerVersions": 3, - "customTimeBefore": _datetime_to_rfc3339(custom_time_before), + "customTimeBefore": custom_time_before.isoformat(), } self.assertEqual(dict(conditions), expected) @@ -129,13 +125,8 @@ def test_ctor_w_custom_time_before(self): def test_from_api_repr(self): import datetime - import pytz - from google.cloud._helpers import _datetime_to_rfc3339 - - custom_time_before = datetime.datetime.utcnow().replace( - tzinfo=pytz.UTC - ) + datetime.timedelta(days=10) + custom_time_before = datetime.date(2018, 8, 1) before = datetime.date(2018, 8, 1) klass = self._get_target_class() resource = { @@ -145,7 +136,7 @@ def test_from_api_repr(self): "matchesStorageClass": ["COLDLINE"], "numNewerVersions": 3, "daysSinceCustomTime": 2, - "customTimeBefore": _datetime_to_rfc3339(custom_time_before), + "customTimeBefore": custom_time_before.isoformat(), } conditions = klass.from_api_repr(resource) self.assertEqual(conditions.age, 10) From ae71f29fa68f56b69dd8eab8c8f7200de9ca26c2 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Mon, 17 Aug 2020 11:47:26 +0530 Subject: [PATCH 7/8] feat: change custom_time to datetime --- google/cloud/storage/blob.py | 15 ++++++++------- tests/unit/test_blob.py | 36 +++++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 86ed6afe6..172499735 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -50,9 +50,9 @@ from google.resumable_media.requests import ResumableUpload from google.api_core.iam import Policy -from google.api_core import datetime_helpers from google.cloud import exceptions from google.cloud._helpers import _bytes_to_unicode +from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud._helpers import _rfc3339_to_datetime from google.cloud._helpers import _to_bytes from google.cloud.exceptions import NotFound @@ -3355,14 +3355,14 @@ def custom_time(self): See https://cloud.google.com/storage/docs/json_api/v1/objects - :rtype: :class:`datetime.date` or ``NoneType`` - :returns: Date object parsed from iso8601 valid date, or + :rtype: :class:`datetime.datetime` or ``NoneType`` + :returns: Datetime object parsed from RFC3339 valid timestamp, or ``None`` if the blob's resource has not been loaded from the server (see :meth:`reload`). """ value = self._properties.get("customTime") if value is not None: - return datetime_helpers.from_iso8601_date(value) + return _rfc3339_to_datetime(value) @custom_time.setter def custom_time(self, value): @@ -3370,11 +3370,12 @@ def custom_time(self, value): See https://cloud.google.com/storage/docs/json_api/v1/objects - :type value: :class:`datetime.date` or ``NoneType`` - :param value: (Optional) Set the custom time of blob. + :type value: :class:`datetime.datetime` or ``NoneType`` + :param value: (Optional) Set the custom time of blob. Datetime object + parsed from RFC3339 valid timestamp. """ if value is not None: - value = value.isoformat() + value = _datetime_to_rfc3339(value) self._properties["customTime"] = value diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 726121ca3..e1afcbe5d 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -134,8 +134,6 @@ def _set_properties_helper(self, kms_key_name=None): CRC32C = "FACE0DAC" COMPONENT_COUNT = 2 ETAG = "ETAG" - custom_time = datetime.date(2018, 8, 1) - CUSTOM_TIME = custom_time.isoformat() resource = { "id": BLOB_ID, "selfLink": SELF_LINK, @@ -159,7 +157,7 @@ def _set_properties_helper(self, kms_key_name=None): "crc32c": CRC32C, "componentCount": COMPONENT_COUNT, "etag": ETAG, - "customTime": CUSTOM_TIME, + "customTime": NOW, } if kms_key_name is not None: @@ -191,7 +189,7 @@ def _set_properties_helper(self, kms_key_name=None): self.assertEqual(blob.crc32c, CRC32C) self.assertEqual(blob.component_count, COMPONENT_COUNT) self.assertEqual(blob.etag, ETAG) - self.assertEqual(blob.custom_time, custom_time) + self.assertEqual(blob.custom_time, now) if kms_key_name is not None: self.assertEqual(blob.kms_key_name, kms_key_name) @@ -4253,31 +4251,39 @@ def test_updated_unset(self): self.assertIsNone(blob.updated) def test_custom_time_getter(self): + from google.cloud._helpers import _RFC3339_MICROS + from google.cloud._helpers import UTC + BLOB_NAME = "blob-name" bucket = _Bucket() - custom_time = datetime.date(2018, 8, 1) - CUSTOM_TIME = custom_time.isoformat() - properties = {"customTime": CUSTOM_TIME} + TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) + TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) + properties = {"customTime": TIME_CREATED} blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.custom_time, custom_time) + self.assertEqual(blob.custom_time, TIMESTAMP) def test_custom_time_setter(self): + from google.cloud._helpers import UTC + BLOB_NAME = "blob-name" bucket = _Bucket() - custom_time = datetime.date(2018, 8, 1) + TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) blob = self._make_one(BLOB_NAME, bucket=bucket) self.assertIsNone(blob.custom_time) - blob.custom_time = custom_time - self.assertEqual(blob.custom_time, custom_time) + blob.custom_time = TIMESTAMP + self.assertEqual(blob.custom_time, TIMESTAMP) def test_custom_time_setter_none_value(self): + from google.cloud._helpers import _RFC3339_MICROS + from google.cloud._helpers import UTC + BLOB_NAME = "blob-name" bucket = _Bucket() - custom_time = datetime.date(2018, 8, 1) - CUSTOM_TIME = custom_time.isoformat() - properties = {"customTime": CUSTOM_TIME} + TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) + TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) + properties = {"customTime": TIME_CREATED} blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.custom_time, custom_time) + self.assertEqual(blob.custom_time, TIMESTAMP) blob.custom_time = None self.assertIsNone(blob.custom_time) From 39a12f0e2c82c87e38d415aa6c73a76eb349a99c Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Tue, 25 Aug 2020 11:01:49 +0530 Subject: [PATCH 8/8] feat: nit --- google/cloud/storage/blob.py | 5 ++++- google/cloud/storage/bucket.py | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 0e28249b3..040a09238 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -3366,7 +3366,10 @@ def custom_time(self): @custom_time.setter def custom_time(self, value): - """Set the custom time for the object. + """Set the custom time for the object. Once set it can't be unset + and only changed to a custom datetime in the future. If the + custom_time must be unset, you must either perform a rewrite operation + or upload the data again. See https://cloud.google.com/storage/docs/json_api/v1/objects diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index b727375f9..9fc77837c 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -178,8 +178,9 @@ class LifecycleRuleConditions(dict): eligible for lifecycle action as soon as it becomes custom. :type custom_time_before: :class:`datetime.date` - :param custom_time_before: (Optional) Apply rule action to items whose custom time is before this - date. This condition is relevant only for versioned objects. + :param custom_time_before: (Optional) Date object parsed from RFC3339 valid date, apply rule action + to items whose custom time is before this date. This condition is relevant + only for versioned objects, e.g., 2019-03-16. :raises ValueError: if no arguments are passed. """