-
- As of January 1, 2020 this library no longer supports Python 2 on the latest released version. +
+ As of January 1, 2020 this library no longer supports Python 2 on the latest released version. Library versions released prior to that date will continue to be available. For more information please visit Python 2 support on Google Cloud.
diff --git a/docs/conf.py b/docs/conf.py index 78e49ed55c..64058683e8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -# Copyright 2024 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# google-cloud-spanner documentation build configuration file +# +# google-cloud-spanner-admin-database documentation build configuration file # # This file is execfile()d with the current directory set to its # containing dir. @@ -42,7 +43,7 @@ # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = "1.5.5" +needs_sphinx = "4.5.0" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -80,8 +81,8 @@ root_doc = "index" # General information about the project. -project = "google-cloud-spanner" -copyright = "2019, Google" +project = "google-cloud-spanner-admin-database" +copyright = "2025, Google, LLC" author = "Google APIs" # The version info for the project you're documenting, acts as replacement for @@ -154,9 +155,9 @@ # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - "description": "Google Cloud Client Libraries for google-cloud-spanner", + "description": "Google Cloud Client Libraries for google-cloud-spanner-admin-database", "github_user": "googleapis", - "github_repo": "python-spanner", + "github_repo": "google-cloud-python", "github_banner": True, "font_family": "'Roboto', Georgia, sans", "head_font_family": "'Roboto', Georgia, serif", @@ -248,7 +249,7 @@ # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = "google-cloud-spanner-doc" +htmlhelp_basename = "google-cloud-spanner-admin-database-doc" # -- Options for warnings ------------------------------------------------------ @@ -266,13 +267,13 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', + # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', + # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. - #'preamble': '', + # 'preamble': '', # Latex figure (float) alignment - #'figure_align': 'htbp', + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples @@ -281,8 +282,8 @@ latex_documents = [ ( root_doc, - "google-cloud-spanner.tex", - "google-cloud-spanner Documentation", + "google-cloud-spanner-admin-database.tex", + "google-cloud-spanner-admin-database Documentation", author, "manual", ) @@ -316,8 +317,8 @@ man_pages = [ ( root_doc, - "google-cloud-spanner", - "google-cloud-spanner Documentation", + "google-cloud-spanner-admin-database", + "google-cloud-spanner-admin-database Documentation", [author], 1, ) @@ -335,11 +336,11 @@ texinfo_documents = [ ( root_doc, - "google-cloud-spanner", - "google-cloud-spanner Documentation", + "google-cloud-spanner-admin-database", + "google-cloud-spanner-admin-database Documentation", author, - "google-cloud-spanner", - "google-cloud-spanner Library", + "google-cloud-spanner-admin-database", + "google-cloud-spanner-admin-database Library", "APIs", ) ] diff --git a/google/cloud/spanner_v1/services/spanner/async_client.py b/google/cloud/spanner_v1/services/spanner/async_client.py index c48b62d532..b197172a8a 100644 --- a/google/cloud/spanner_v1/services/spanner/async_client.py +++ b/google/cloud/spanner_v1/services/spanner/async_client.py @@ -49,6 +49,7 @@ from google.cloud.spanner_v1.services.spanner import pagers from google.cloud.spanner_v1.types import commit_response +from google.cloud.spanner_v1.types import location from google.cloud.spanner_v1.types import mutation from google.cloud.spanner_v1.types import result_set from google.cloud.spanner_v1.types import spanner @@ -477,10 +478,11 @@ async def sample_batch_create_sessions(): should not be set. session_count (:class:`int`): Required. The number of sessions to be created in this - batch call. The API can return fewer than the requested - number of sessions. If a specific number of sessions are - desired, the client can make additional calls to - ``BatchCreateSessions`` (adjusting + batch call. At least one session is created. The API can + return fewer than the requested number of sessions. If a + specific number of sessions are desired, the client can + make additional calls to ``BatchCreateSessions`` + (adjusting [session_count][google.spanner.v1.BatchCreateSessionsRequest.session_count] as necessary). diff --git a/google/cloud/spanner_v1/services/spanner/client.py b/google/cloud/spanner_v1/services/spanner/client.py index 82dbf8375e..d542dd89ef 100644 --- a/google/cloud/spanner_v1/services/spanner/client.py +++ b/google/cloud/spanner_v1/services/spanner/client.py @@ -64,6 +64,7 @@ from google.cloud.spanner_v1.services.spanner import pagers from google.cloud.spanner_v1.types import commit_response +from google.cloud.spanner_v1.types import location from google.cloud.spanner_v1.types import mutation from google.cloud.spanner_v1.types import result_set from google.cloud.spanner_v1.types import spanner @@ -922,10 +923,11 @@ def sample_batch_create_sessions(): should not be set. session_count (int): Required. The number of sessions to be created in this - batch call. The API can return fewer than the requested - number of sessions. If a specific number of sessions are - desired, the client can make additional calls to - ``BatchCreateSessions`` (adjusting + batch call. At least one session is created. The API can + return fewer than the requested number of sessions. If a + specific number of sessions are desired, the client can + make additional calls to ``BatchCreateSessions`` + (adjusting [session_count][google.spanner.v1.BatchCreateSessionsRequest.session_count] as necessary). diff --git a/google/cloud/spanner_v1/types/__init__.py b/google/cloud/spanner_v1/types/__init__.py index e2f87d65da..5a7ded16dd 100644 --- a/google/cloud/spanner_v1/types/__init__.py +++ b/google/cloud/spanner_v1/types/__init__.py @@ -23,11 +23,21 @@ KeyRange, KeySet, ) +from .location import ( + CacheUpdate, + Group, + KeyRecipe, + Range, + RecipeList, + RoutingHint, + Tablet, +) from .mutation import ( Mutation, ) from .query_plan import ( PlanNode, + QueryAdvisorResult, QueryPlan, ) from .result_set import ( @@ -80,8 +90,16 @@ "CommitResponse", "KeyRange", "KeySet", + "CacheUpdate", + "Group", + "KeyRecipe", + "Range", + "RecipeList", + "RoutingHint", + "Tablet", "Mutation", "PlanNode", + "QueryAdvisorResult", "QueryPlan", "PartialResultSet", "ResultSet", diff --git a/google/cloud/spanner_v1/types/location.py b/google/cloud/spanner_v1/types/location.py new file mode 100644 index 0000000000..1749e87aef --- /dev/null +++ b/google/cloud/spanner_v1/types/location.py @@ -0,0 +1,677 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.cloud.spanner_v1.types import type as gs_type +from google.protobuf import struct_pb2 # type: ignore + + +__protobuf__ = proto.module( + package="google.spanner.v1", + manifest={ + "Range", + "Tablet", + "Group", + "KeyRecipe", + "RecipeList", + "CacheUpdate", + "RoutingHint", + }, +) + + +class Range(proto.Message): + r"""A ``Range`` represents a range of keys in a database. The keys + themselves are encoded in "sortable string format", also known as + ssformat. Consult Spanner's open source client libraries for details + on the encoding. + + Each range represents a contiguous range of rows, possibly from + multiple tables/indexes. Each range is associated with a single + paxos group (known as a "group" throughout this API), a split (which + names the exact range within the group), and a generation that can + be used to determine whether a given ``Range`` represents a newer or + older location for the key range. + + Attributes: + start_key (bytes): + The start key of the range, inclusive. + Encoded in "sortable string format" (ssformat). + limit_key (bytes): + The limit key of the range, exclusive. + Encoded in "sortable string format" (ssformat). + group_uid (int): + The UID of the paxos group where this range is stored. UIDs + are unique within the database. References + ``Group.group_uid``. + split_id (int): + A group can store multiple ranges of keys. Each key range is + named by an ID (the split ID). Within a group, split IDs are + unique. The ``split_id`` names the exact split in + ``group_uid`` where this range is stored. + generation (bytes): + ``generation`` indicates the freshness of the range + information contained in this proto. Generations can be + compared lexicographically; if generation A is greater than + generation B, then the ``Range`` corresponding to A is newer + than the ``Range`` corresponding to B, and should be used + preferentially. + """ + + start_key: bytes = proto.Field( + proto.BYTES, + number=1, + ) + limit_key: bytes = proto.Field( + proto.BYTES, + number=2, + ) + group_uid: int = proto.Field( + proto.UINT64, + number=3, + ) + split_id: int = proto.Field( + proto.UINT64, + number=4, + ) + generation: bytes = proto.Field( + proto.BYTES, + number=5, + ) + + +class Tablet(proto.Message): + r"""A ``Tablet`` represents a single replica of a ``Group``. A tablet is + served by a single server at a time, and can move between servers + due to server death or simply load balancing. + + Attributes: + tablet_uid (int): + The UID of the tablet, unique within the database. Matches + the ``tablet_uids`` and ``leader_tablet_uid`` fields in + ``Group``. + server_address (str): + The address of the server that is serving + this tablet -- either an IP address or DNS + hostname and a port number. + location (str): + Where this tablet is located. In the Spanner + managed service, this is the name of a region, + such as "us-central1". In Spanner Omni, this is + a previously created location. + role (google.cloud.spanner_v1.types.Tablet.Role): + The role of the tablet. + incarnation (bytes): + ``incarnation`` indicates the freshness of the tablet + information contained in this proto. Incarnations can be + compared lexicographically; if incarnation A is greater than + incarnation B, then the ``Tablet`` corresponding to A is + newer than the ``Tablet`` corresponding to B, and should be + used preferentially. + distance (int): + Distances help the client pick the closest tablet out of the + list of tablets for a given request. Tablets with lower + distances should generally be preferred. Tablets with the + same distance are approximately equally close; the client + can choose arbitrarily. + + Distances do not correspond precisely to expected latency, + geographical distance, or anything else. Distances should be + compared only between tablets of the same group; they are + not meaningful between different groups. + + A value of zero indicates that the tablet may be in the same + zone as the client, and have minimum network latency. A + value less than or equal to five indicates that the tablet + is thought to be in the same region as the client, and may + have a few milliseconds of network latency. Values greater + than five are most likely in a different region, with + non-trivial network latency. + + Clients should use the following algorithm: + + - If the request is using a directed read, eliminate any + tablets that do not match the directed read's target zone + and/or replica type. + - (Read-write transactions only) Choose leader tablet if it + has an distance <=5. + - Group and sort tablets by distance. Choose a random tablet + with the lowest distance. If the request is not a directed + read, only consider replicas with distances <=5. + - Send the request to the fallback endpoint. + + The tablet picked by this algorithm may be skipped, either + because it is marked as ``skip`` by the server or because + the corresponding server is unreachable, flow controlled, + etc. Skipped tablets should be added to the + ``skipped_tablet_uid`` field in ``RoutingHint``; the + algorithm above should then be re-run without including the + skipped tablet(s) to pick the next best tablet. + skip (bool): + If true, the tablet should not be chosen by the client. + Typically, this signals that the tablet is unhealthy in some + way. Tablets with ``skip`` set to true should be reported + back to the server in ``RoutingHint.skipped_tablet_uid``; + this cues the server to send updated information for this + tablet should it become usable again. + """ + + class Role(proto.Enum): + r"""Indicates the role of the tablet. + + Values: + ROLE_UNSPECIFIED (0): + Not specified. + READ_WRITE (1): + The tablet can perform reads and (if elected + leader) writes. + READ_ONLY (2): + The tablet can only perform reads. + """ + ROLE_UNSPECIFIED = 0 + READ_WRITE = 1 + READ_ONLY = 2 + + tablet_uid: int = proto.Field( + proto.UINT64, + number=1, + ) + server_address: str = proto.Field( + proto.STRING, + number=2, + ) + location: str = proto.Field( + proto.STRING, + number=3, + ) + role: Role = proto.Field( + proto.ENUM, + number=4, + enum=Role, + ) + incarnation: bytes = proto.Field( + proto.BYTES, + number=5, + ) + distance: int = proto.Field( + proto.UINT32, + number=6, + ) + skip: bool = proto.Field( + proto.BOOL, + number=7, + ) + + +class Group(proto.Message): + r"""A ``Group`` represents a paxos group in a database. A group is a set + of tablets that are replicated across multiple servers. Groups may + have a leader tablet. Groups store one (or sometimes more) ranges of + keys. + + Attributes: + group_uid (int): + The UID of the paxos group, unique within the database. + Matches the ``group_uid`` field in ``Range``. + tablets (MutableSequence[google.cloud.spanner_v1.types.Tablet]): + A list of tablets that are part of the group. Note that this + list may not be exhaustive; it will only include tablets the + server considers useful to the client. The returned list is + ordered ascending by distance. + + Tablet UIDs reference ``Tablet.tablet_uid``. + leader_index (int): + The last known leader tablet of the group as an index into + ``tablets``. May be negative if the group has no known + leader. + generation (bytes): + ``generation`` indicates the freshness of the group + information (including leader information) contained in this + proto. Generations can be compared lexicographically; if + generation A is greater than generation B, then the + ``Group`` corresponding to A is newer than the ``Group`` + corresponding to B, and should be used preferentially. + """ + + group_uid: int = proto.Field( + proto.UINT64, + number=1, + ) + tablets: MutableSequence["Tablet"] = proto.RepeatedField( + proto.MESSAGE, + number=2, + message="Tablet", + ) + leader_index: int = proto.Field( + proto.INT32, + number=3, + ) + generation: bytes = proto.Field( + proto.BYTES, + number=4, + ) + + +class KeyRecipe(proto.Message): + r"""A ``KeyRecipe`` provides the metadata required to translate reads, + mutations, and queries into a byte array in "sortable string format" + (ssformat)that can be used with ``Range``\ s to route requests. Note + that the client *must* tolerate ``KeyRecipe``\ s that appear to be + invalid, since the ``KeyRecipe`` format may change over time. + Requests with invalid ``KeyRecipe``\ s should be routed to a default + server. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + table_name (str): + A table name, matching the name from the + database schema. + + This field is a member of `oneof`_ ``target``. + index_name (str): + An index name, matching the name from the + database schema. + + This field is a member of `oneof`_ ``target``. + operation_uid (int): + The UID of a query, matching the UID from ``RoutingHint``. + + This field is a member of `oneof`_ ``target``. + part (MutableSequence[google.cloud.spanner_v1.types.KeyRecipe.Part]): + Parts are in the order they should appear in + the encoded key. + """ + + class Part(proto.Message): + r"""An ssformat key is composed of a sequence of tag numbers and key + column values. ``Part`` represents a single tag or key column value. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + tag (int): + If non-zero, ``tag`` is the only field present in this + ``Part``. The part is encoded by appending ``tag`` to the + ssformat key. + order (google.cloud.spanner_v1.types.KeyRecipe.Part.Order): + Whether the key column is sorted ascending or descending. + Only present if ``tag`` is zero. + null_order (google.cloud.spanner_v1.types.KeyRecipe.Part.NullOrder): + How NULLs are represented in the encoded key part. Only + present if ``tag`` is zero. + type_ (google.cloud.spanner_v1.types.Type): + The type of the key part. Only present if ``tag`` is zero. + identifier (str): + ``identifier`` is the name of the column or query parameter. + + This field is a member of `oneof`_ ``value_type``. + value (google.protobuf.struct_pb2.Value): + The constant value of the key part. + It is present when query uses a constant as a + part of the key. + + This field is a member of `oneof`_ ``value_type``. + random (bool): + If true, the client is responsible to fill in + the value randomly. It's relevant only for the + INT64 type. + + This field is a member of `oneof`_ ``value_type``. + struct_identifiers (MutableSequence[int]): + It is a repeated field to support fetching key columns from + nested structs, such as ``STRUCT`` query parameters. + """ + + class Order(proto.Enum): + r"""The remaining fields encode column values. + + Values: + ORDER_UNSPECIFIED (0): + Default value, equivalent to ``ASCENDING``. + ASCENDING (1): + The key is ascending - corresponds to ``ASC`` in the schema + definition. + DESCENDING (2): + The key is descending - corresponds to ``DESC`` in the + schema definition. + """ + ORDER_UNSPECIFIED = 0 + ASCENDING = 1 + DESCENDING = 2 + + class NullOrder(proto.Enum): + r"""The null order of the key column. This dictates where NULL values + sort in the sorted order. Note that columns which are ``NOT NULL`` + can have a special encoding. + + Values: + NULL_ORDER_UNSPECIFIED (0): + Default value. This value is unused. + NULLS_FIRST (1): + NULL values sort before any non-NULL values. + NULLS_LAST (2): + NULL values sort after any non-NULL values. + NOT_NULL (3): + The column does not support NULL values. + """ + NULL_ORDER_UNSPECIFIED = 0 + NULLS_FIRST = 1 + NULLS_LAST = 2 + NOT_NULL = 3 + + tag: int = proto.Field( + proto.UINT32, + number=1, + ) + order: "KeyRecipe.Part.Order" = proto.Field( + proto.ENUM, + number=2, + enum="KeyRecipe.Part.Order", + ) + null_order: "KeyRecipe.Part.NullOrder" = proto.Field( + proto.ENUM, + number=3, + enum="KeyRecipe.Part.NullOrder", + ) + type_: gs_type.Type = proto.Field( + proto.MESSAGE, + number=4, + message=gs_type.Type, + ) + identifier: str = proto.Field( + proto.STRING, + number=5, + oneof="value_type", + ) + value: struct_pb2.Value = proto.Field( + proto.MESSAGE, + number=6, + oneof="value_type", + message=struct_pb2.Value, + ) + random: bool = proto.Field( + proto.BOOL, + number=8, + oneof="value_type", + ) + struct_identifiers: MutableSequence[int] = proto.RepeatedField( + proto.INT32, + number=7, + ) + + table_name: str = proto.Field( + proto.STRING, + number=1, + oneof="target", + ) + index_name: str = proto.Field( + proto.STRING, + number=2, + oneof="target", + ) + operation_uid: int = proto.Field( + proto.UINT64, + number=3, + oneof="target", + ) + part: MutableSequence[Part] = proto.RepeatedField( + proto.MESSAGE, + number=4, + message=Part, + ) + + +class RecipeList(proto.Message): + r"""A ``RecipeList`` contains a list of ``KeyRecipe``\ s, which share + the same schema generation. + + Attributes: + schema_generation (bytes): + The schema generation of the recipes. To be sent to the + server in ``RoutingHint.schema_generation`` whenever one of + the recipes is used. ``schema_generation`` values are + comparable with each other; if generation A compares greater + than generation B, then A is a more recent schema than B. + Clients should in general aim to cache only the latest + schema generation, and discard more stale recipes. + recipe (MutableSequence[google.cloud.spanner_v1.types.KeyRecipe]): + A list of recipes to be cached. + """ + + schema_generation: bytes = proto.Field( + proto.BYTES, + number=1, + ) + recipe: MutableSequence["KeyRecipe"] = proto.RepeatedField( + proto.MESSAGE, + number=3, + message="KeyRecipe", + ) + + +class CacheUpdate(proto.Message): + r"""A ``CacheUpdate`` expresses a set of changes the client should + incorporate into its location cache. These changes may or may not be + newer than what the client has in its cache, and should be discarded + if necessary. ``CacheUpdate``\ s can be obtained in response to + requests that included a ``RoutingHint`` field, but may also be + obtained by explicit location-fetching RPCs which may be added in + the future. + + Attributes: + database_id (int): + An internal ID for the database. Database + names can be reused if a database is deleted and + re-created. Each time the database is + re-created, it will get a new database ID, which + will never be re-used for any other database. + range_ (MutableSequence[google.cloud.spanner_v1.types.Range]): + A list of ranges to be cached. + group (MutableSequence[google.cloud.spanner_v1.types.Group]): + A list of groups to be cached. + key_recipes (google.cloud.spanner_v1.types.RecipeList): + A list of recipes to be cached. + """ + + database_id: int = proto.Field( + proto.UINT64, + number=1, + ) + range_: MutableSequence["Range"] = proto.RepeatedField( + proto.MESSAGE, + number=2, + message="Range", + ) + group: MutableSequence["Group"] = proto.RepeatedField( + proto.MESSAGE, + number=3, + message="Group", + ) + key_recipes: "RecipeList" = proto.Field( + proto.MESSAGE, + number=5, + message="RecipeList", + ) + + +class RoutingHint(proto.Message): + r"""``RoutingHint`` can be optionally added to location-aware Spanner + requests. It gives the server hints that can be used to route the + request to an appropriate server, potentially significantly + decreasing latency and improving throughput. To achieve improved + performance, most fields must be filled in with accurate values. + + The presence of a valid ``RoutingHint`` tells the server that the + client is location-aware. + + ``RoutingHint`` does not change the semantics of the request; it is + purely a performance hint; the request will perform the same actions + on the database's data as if ``RoutingHint`` were not present. + However, if the ``RoutingHint`` is incomplete or incorrect, the + response may include a ``CacheUpdate`` the client can use to correct + its location cache. + + Attributes: + operation_uid (int): + A session-scoped unique ID for the operation, computed + client-side. Requests with the same ``operation_uid`` should + have a shared 'shape', meaning that some fields are expected + to be the same, such as the SQL query, the target + table/columns (for reads) etc. Requests with the same + ``operation_uid`` are meant to differ only in fields like + keys/key ranges/query parameters, transaction IDs, etc. + + ``operation_uid`` must be non-zero for ``RoutingHint`` to be + valid. + database_id (int): + The database ID of the database being accessed, see + ``CacheUpdate.database_id``. Should match the cache entries + that were used to generate the rest of the fields in this + ``RoutingHint``. + schema_generation (bytes): + The schema generation of the recipe that was used to + generate ``key`` and ``limit_key``. See also + ``RecipeList.schema_generation``. + key (bytes): + The key / key range that this request accesses. For + operations that access a single key, ``key`` should be set + and ``limit_key`` should be empty. For operations that + access a key range, ``key`` and ``limit_key`` should both be + set, to the inclusive start and exclusive end of the range + respectively. + + The keys are encoded in "sortable string format" (ssformat), + using a ``KeyRecipe`` that is appropriate for the request. + See ``KeyRecipe`` for more details. + limit_key (bytes): + If this request targets a key range, this is the exclusive + end of the range. See ``key`` for more details. + group_uid (int): + The group UID of the group that the client believes serves + the range defined by ``key`` and ``limit_key``. See + ``Range.group_uid`` for more details. + split_id (int): + The split ID of the split that the client believes contains + the range defined by ``key`` and ``limit_key``. See + ``Range.split_id`` for more details. + tablet_uid (int): + The tablet UID of the tablet from group ``group_uid`` that + the client believes is best to serve this request. See + ``Group.local_tablet_uids`` and ``Group.leader_tablet_uid``. + skipped_tablet_uid (MutableSequence[google.cloud.spanner_v1.types.RoutingHint.SkippedTablet]): + If the client had multiple options for tablet selection, and + some of its first choices were unhealthy (e.g., the server + is unreachable, or ``Tablet.skip`` is true), this field will + contain the tablet UIDs of those tablets, with their + incarnations. The server may include a ``CacheUpdate`` with + new locations for those tablets. + client_location (str): + If present, the client's current location. In + the Spanner managed service, this should be the + name of a Google Cloud zone or region, such as + "us-central1". In Spanner Omni, this should + correspond to a previously created location. + + If absent, the client's location will be assumed + to be the same as the location of the server the + client ends up connected to. + + Locations are primarily valuable for clients + that connect from regions other than the ones + that contain the Spanner database. + """ + + class SkippedTablet(proto.Message): + r"""A tablet that was skipped by the client. See ``Tablet.tablet_uid`` + and ``Tablet.incarnation``. + + Attributes: + tablet_uid (int): + The tablet UID of the tablet that was skipped. See + ``Tablet.tablet_uid``. + incarnation (bytes): + The incarnation of the tablet that was skipped. See + ``Tablet.incarnation``. + """ + + tablet_uid: int = proto.Field( + proto.UINT64, + number=1, + ) + incarnation: bytes = proto.Field( + proto.BYTES, + number=2, + ) + + operation_uid: int = proto.Field( + proto.UINT64, + number=1, + ) + database_id: int = proto.Field( + proto.UINT64, + number=2, + ) + schema_generation: bytes = proto.Field( + proto.BYTES, + number=3, + ) + key: bytes = proto.Field( + proto.BYTES, + number=4, + ) + limit_key: bytes = proto.Field( + proto.BYTES, + number=5, + ) + group_uid: int = proto.Field( + proto.UINT64, + number=6, + ) + split_id: int = proto.Field( + proto.UINT64, + number=7, + ) + tablet_uid: int = proto.Field( + proto.UINT64, + number=8, + ) + skipped_tablet_uid: MutableSequence[SkippedTablet] = proto.RepeatedField( + proto.MESSAGE, + number=9, + message=SkippedTablet, + ) + client_location: str = proto.Field( + proto.STRING, + number=10, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/spanner_v1/types/mutation.py b/google/cloud/spanner_v1/types/mutation.py index 8389910fc0..3cbc3b937b 100644 --- a/google/cloud/spanner_v1/types/mutation.py +++ b/google/cloud/spanner_v1/types/mutation.py @@ -21,6 +21,7 @@ from google.cloud.spanner_v1.types import keys from google.protobuf import struct_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore __protobuf__ = proto.module( @@ -89,6 +90,14 @@ class Mutation(proto.Message): Delete rows from a table. Succeeds whether or not the named rows were present. + This field is a member of `oneof`_ ``operation``. + send (google.cloud.spanner_v1.types.Mutation.Send): + Send a message to a queue. + + This field is a member of `oneof`_ ``operation``. + ack (google.cloud.spanner_v1.types.Mutation.Ack): + Ack a message from a queue. + This field is a member of `oneof`_ ``operation``. """ @@ -166,6 +175,79 @@ class Delete(proto.Message): message=keys.KeySet, ) + class Send(proto.Message): + r"""Arguments to [send][google.spanner.v1.Mutation.send] operations. + + Attributes: + queue (str): + Required. The queue to which the message will + be sent. + key (google.protobuf.struct_pb2.ListValue): + Required. The primary key of the message to + be sent. + deliver_time (google.protobuf.timestamp_pb2.Timestamp): + The time at which Spanner will begin attempting to deliver + the message. If ``deliver_time`` is not set, Spanner will + deliver the message immediately. If ``deliver_time`` is in + the past, Spanner will replace it with a value closer to the + current time. + payload (google.protobuf.struct_pb2.Value): + The payload of the message. + """ + + queue: str = proto.Field( + proto.STRING, + number=1, + ) + key: struct_pb2.ListValue = proto.Field( + proto.MESSAGE, + number=2, + message=struct_pb2.ListValue, + ) + deliver_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=3, + message=timestamp_pb2.Timestamp, + ) + payload: struct_pb2.Value = proto.Field( + proto.MESSAGE, + number=4, + message=struct_pb2.Value, + ) + + class Ack(proto.Message): + r"""Arguments to [ack][google.spanner.v1.Mutation.ack] operations. + + Attributes: + queue (str): + Required. The queue where the message to be + acked is stored. + key (google.protobuf.struct_pb2.ListValue): + Required. The primary key of the message to + be acked. + ignore_not_found (bool): + By default, an attempt to ack a message that does not exist + will fail with a ``NOT_FOUND`` error. With + ``ignore_not_found`` set to true, the ack will succeed even + if the message does not exist. This is useful for + unconditionally acking a message, even if it is missing or + has already been acked. + """ + + queue: str = proto.Field( + proto.STRING, + number=1, + ) + key: struct_pb2.ListValue = proto.Field( + proto.MESSAGE, + number=2, + message=struct_pb2.ListValue, + ) + ignore_not_found: bool = proto.Field( + proto.BOOL, + number=3, + ) + insert: Write = proto.Field( proto.MESSAGE, number=1, @@ -196,6 +278,18 @@ class Delete(proto.Message): oneof="operation", message=Delete, ) + send: Send = proto.Field( + proto.MESSAGE, + number=6, + oneof="operation", + message=Send, + ) + ack: Ack = proto.Field( + proto.MESSAGE, + number=7, + oneof="operation", + message=Ack, + ) __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/spanner_v1/types/query_plan.py b/google/cloud/spanner_v1/types/query_plan.py index d361911f1d..efe32934f8 100644 --- a/google/cloud/spanner_v1/types/query_plan.py +++ b/google/cloud/spanner_v1/types/query_plan.py @@ -26,6 +26,7 @@ package="google.spanner.v1", manifest={ "PlanNode", + "QueryAdvisorResult", "QueryPlan", }, ) @@ -198,6 +199,49 @@ class ShortRepresentation(proto.Message): ) +class QueryAdvisorResult(proto.Message): + r"""Output of query advisor analysis. + + Attributes: + index_advice (MutableSequence[google.cloud.spanner_v1.types.QueryAdvisorResult.IndexAdvice]): + Optional. Index Recommendation for a query. + This is an optional field and the recommendation + will only be available when the recommendation + guarantees significant improvement in query + performance. + """ + + class IndexAdvice(proto.Message): + r"""Recommendation to add new indexes to run queries more + efficiently. + + Attributes: + ddl (MutableSequence[str]): + Optional. DDL statements to add new indexes + that will improve the query. + improvement_factor (float): + Optional. Estimated latency improvement + factor. For example if the query currently takes + 500 ms to run and the estimated latency with new + indexes is 100 ms this field will be 5. + """ + + ddl: MutableSequence[str] = proto.RepeatedField( + proto.STRING, + number=1, + ) + improvement_factor: float = proto.Field( + proto.DOUBLE, + number=2, + ) + + index_advice: MutableSequence[IndexAdvice] = proto.RepeatedField( + proto.MESSAGE, + number=1, + message=IndexAdvice, + ) + + class QueryPlan(proto.Message): r"""Contains an ordered list of nodes appearing in the query plan. @@ -208,6 +252,10 @@ class QueryPlan(proto.Message): pre-order starting with the plan root. Each [PlanNode][google.spanner.v1.PlanNode]'s ``id`` corresponds to its index in ``plan_nodes``. + query_advice (google.cloud.spanner_v1.types.QueryAdvisorResult): + Optional. The advise/recommendations for a + query. Currently this field will be serving + index recommendations for a query. """ plan_nodes: MutableSequence["PlanNode"] = proto.RepeatedField( @@ -215,6 +263,11 @@ class QueryPlan(proto.Message): number=1, message="PlanNode", ) + query_advice: "QueryAdvisorResult" = proto.Field( + proto.MESSAGE, + number=2, + message="QueryAdvisorResult", + ) __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/spanner_v1/types/result_set.py b/google/cloud/spanner_v1/types/result_set.py index 697d0fd33b..0ab386bc61 100644 --- a/google/cloud/spanner_v1/types/result_set.py +++ b/google/cloud/spanner_v1/types/result_set.py @@ -19,6 +19,7 @@ import proto # type: ignore +from google.cloud.spanner_v1.types import location from google.cloud.spanner_v1.types import query_plan as gs_query_plan from google.cloud.spanner_v1.types import transaction as gs_transaction from google.cloud.spanner_v1.types import type as gs_type @@ -223,6 +224,14 @@ class PartialResultSet(proto.Message): ``PartialResultSet`` in the stream. The server might optionally set this field. Clients shouldn't rely on this field being set in all cases. + cache_update (google.cloud.spanner_v1.types.CacheUpdate): + Optional. A cache update expresses a set of changes the + client should incorporate into its location cache. The + client should discard the changes if they are older than the + data it already has. This data can be obtained in response + to requests that included a ``RoutingHint`` field, but may + also be obtained by explicit location-fetching RPCs which + may be added in the future. """ metadata: "ResultSetMetadata" = proto.Field( @@ -257,6 +266,11 @@ class PartialResultSet(proto.Message): proto.BOOL, number=9, ) + cache_update: location.CacheUpdate = proto.Field( + proto.MESSAGE, + number=10, + message=location.CacheUpdate, + ) class ResultSetMetadata(proto.Message): diff --git a/google/cloud/spanner_v1/types/spanner.py b/google/cloud/spanner_v1/types/spanner.py index 9e7a477b46..6e363088de 100644 --- a/google/cloud/spanner_v1/types/spanner.py +++ b/google/cloud/spanner_v1/types/spanner.py @@ -20,6 +20,7 @@ import proto # type: ignore from google.cloud.spanner_v1.types import keys +from google.cloud.spanner_v1.types import location as gs_location from google.cloud.spanner_v1.types import mutation from google.cloud.spanner_v1.types import result_set from google.cloud.spanner_v1.types import transaction as gs_transaction @@ -96,10 +97,10 @@ class BatchCreateSessionsRequest(proto.Message): Parameters to apply to each created session. session_count (int): Required. The number of sessions to be created in this batch - call. The API can return fewer than the requested number of - sessions. If a specific number of sessions are desired, the - client can make additional calls to ``BatchCreateSessions`` - (adjusting + call. At least one session is created. The API can return + fewer than the requested number of sessions. If a specific + number of sessions are desired, the client can make + additional calls to ``BatchCreateSessions`` (adjusting [session_count][google.spanner.v1.BatchCreateSessionsRequest.session_count] as necessary). """ @@ -167,9 +168,9 @@ class Session(proto.Message): The database role which created this session. multiplexed (bool): Optional. If ``true``, specifies a multiplexed session. Use - a multiplexed session for multiple, concurrent read-only - operations. Don't use them for read-write transactions, - partitioned reads, or partitioned queries. Use + a multiplexed session for multiple, concurrent operations + including any combination of read-only and read-write + transactions. Use [``sessions.create``][google.spanner.v1.Spanner.CreateSession] to create multiplexed sessions. Don't use [BatchCreateSessions][google.spanner.v1.Spanner.BatchCreateSessions] @@ -660,6 +661,15 @@ class ExecuteSqlRequest(proto.Message): example, validation of unique constraints). Given this, successful execution of a DML statement shouldn't be assumed until a subsequent ``Commit`` call completes successfully. + routing_hint (google.cloud.spanner_v1.types.RoutingHint): + Optional. If present, it makes the Spanner + requests location-aware. + It gives the server hints that can be used to + route the request to an appropriate server, + potentially significantly decreasing latency and + improving throughput. To achieve improved + performance, most fields must be filled in with + accurate values. """ class QueryMode(proto.Enum): @@ -826,6 +836,11 @@ class QueryOptions(proto.Message): proto.BOOL, number=17, ) + routing_hint: gs_location.RoutingHint = proto.Field( + proto.MESSAGE, + number=18, + message=gs_location.RoutingHint, + ) class ExecuteBatchDmlRequest(proto.Message): @@ -1385,6 +1400,15 @@ class ReadRequest(proto.Message): lock_hint (google.cloud.spanner_v1.types.ReadRequest.LockHint): Optional. Lock Hint for the request, it can only be used with read-write transactions. + routing_hint (google.cloud.spanner_v1.types.RoutingHint): + Optional. If present, it makes the Spanner + requests location-aware. + It gives the server hints that can be used to + route the request to an appropriate server, + potentially significantly decreasing latency and + improving throughput. To achieve improved + performance, most fields must be filled in with + accurate values. """ class OrderBy(proto.Enum): @@ -1530,6 +1554,11 @@ class LockHint(proto.Enum): number=17, enum=LockHint, ) + routing_hint: gs_location.RoutingHint = proto.Field( + proto.MESSAGE, + number=18, + message=gs_location.RoutingHint, + ) class BeginTransactionRequest(proto.Message): diff --git a/google/cloud/spanner_v1/types/transaction.py b/google/cloud/spanner_v1/types/transaction.py index 447c310548..0cc11a73a6 100644 --- a/google/cloud/spanner_v1/types/transaction.py +++ b/google/cloud/spanner_v1/types/transaction.py @@ -96,8 +96,9 @@ class TransactionOptions(proto.Message): """ class IsolationLevel(proto.Enum): - r"""``IsolationLevel`` is used when setting ``isolation_level`` for a - transaction. + r"""``IsolationLevel`` is used when setting the `isolation + level `__ + for a transaction. Values: ISOLATION_LEVEL_UNSPECIFIED (0): @@ -124,8 +125,8 @@ class IsolationLevel(proto.Enum): ``SERIALIZABLE`` transactions, only write-write conflicts are detected in snapshot transactions. - This isolation level does not support Read-only and - Partitioned DML transactions. + This isolation level does not support read-only and + partitioned DML transactions. When ``REPEATABLE_READ`` is specified on a read-write transaction, the locking semantics default to diff --git a/scripts/fixup_spanner_v1_keywords.py b/scripts/fixup_spanner_v1_keywords.py index c7f41be11e..e0787f13b4 100644 --- a/scripts/fixup_spanner_v1_keywords.py +++ b/scripts/fixup_spanner_v1_keywords.py @@ -46,15 +46,15 @@ class spannerCallTransformer(cst.CSTTransformer): 'create_session': ('database', 'session', ), 'delete_session': ('name', ), 'execute_batch_dml': ('session', 'transaction', 'statements', 'seqno', 'request_options', 'last_statements', ), - 'execute_sql': ('session', 'sql', 'transaction', 'params', 'param_types', 'resume_token', 'query_mode', 'partition_token', 'seqno', 'query_options', 'request_options', 'directed_read_options', 'data_boost_enabled', 'last_statement', ), - 'execute_streaming_sql': ('session', 'sql', 'transaction', 'params', 'param_types', 'resume_token', 'query_mode', 'partition_token', 'seqno', 'query_options', 'request_options', 'directed_read_options', 'data_boost_enabled', 'last_statement', ), + 'execute_sql': ('session', 'sql', 'transaction', 'params', 'param_types', 'resume_token', 'query_mode', 'partition_token', 'seqno', 'query_options', 'request_options', 'directed_read_options', 'data_boost_enabled', 'last_statement', 'routing_hint', ), + 'execute_streaming_sql': ('session', 'sql', 'transaction', 'params', 'param_types', 'resume_token', 'query_mode', 'partition_token', 'seqno', 'query_options', 'request_options', 'directed_read_options', 'data_boost_enabled', 'last_statement', 'routing_hint', ), 'get_session': ('name', ), 'list_sessions': ('database', 'page_size', 'page_token', 'filter', ), 'partition_query': ('session', 'sql', 'transaction', 'params', 'param_types', 'partition_options', ), 'partition_read': ('session', 'table', 'key_set', 'transaction', 'index', 'columns', 'partition_options', ), - 'read': ('session', 'table', 'columns', 'key_set', 'transaction', 'index', 'limit', 'resume_token', 'partition_token', 'request_options', 'directed_read_options', 'data_boost_enabled', 'order_by', 'lock_hint', ), + 'read': ('session', 'table', 'columns', 'key_set', 'transaction', 'index', 'limit', 'resume_token', 'partition_token', 'request_options', 'directed_read_options', 'data_boost_enabled', 'order_by', 'lock_hint', 'routing_hint', ), 'rollback': ('session', 'transaction_id', ), - 'streaming_read': ('session', 'table', 'columns', 'key_set', 'transaction', 'index', 'limit', 'resume_token', 'partition_token', 'request_options', 'directed_read_options', 'data_boost_enabled', 'order_by', 'lock_hint', ), + 'streaming_read': ('session', 'table', 'columns', 'key_set', 'transaction', 'index', 'limit', 'resume_token', 'partition_token', 'request_options', 'directed_read_options', 'data_boost_enabled', 'order_by', 'lock_hint', 'routing_hint', ), } def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: diff --git a/tests/unit/gapic/spanner_v1/test_spanner.py b/tests/unit/gapic/spanner_v1/test_spanner.py index 80cb748024..d71d85a443 100644 --- a/tests/unit/gapic/spanner_v1/test_spanner.py +++ b/tests/unit/gapic/spanner_v1/test_spanner.py @@ -59,6 +59,7 @@ from google.cloud.spanner_v1.services.spanner import transports from google.cloud.spanner_v1.types import commit_response from google.cloud.spanner_v1.types import keys +from google.cloud.spanner_v1.types import location from google.cloud.spanner_v1.types import mutation from google.cloud.spanner_v1.types import result_set from google.cloud.spanner_v1.types import spanner From 4bb46222180be4eb607fd0307ed827c7619b6e94 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 18 Nov 2025 20:27:16 -0800 Subject: [PATCH 357/364] chore: onboard to librarian (#1454) Towards https://github.com/googleapis/librarian/issues/2460 --- .github/.OwlBot.lock.yaml | 17 - .github/.OwlBot.yaml | 30 - .github/auto-approve.yml | 3 - .github/release-please.yml | 15 - .github/release-trigger.yml | 2 - .github/snippet-bot.yml | 0 .github/sync-repo-settings.yaml | 15 - .github/workflows/presubmit.yaml | 2 +- .../generator-input/.repo-metadata.json | 18 + .../generator-input/librarian.py | 106 +--- .librarian/generator-input/noxfile.py | 595 ++++++++++++++++++ .librarian/generator-input/setup.py | 103 +++ .librarian/state.yaml | 54 ++ docs/conf.py | 28 +- .../gapic_version.py | 2 +- .../gapic_version.py | 2 +- google/cloud/spanner_dbapi/version.py | 4 +- google/cloud/spanner_v1/gapic_version.py | 2 +- noxfile.py | 6 +- release-please-config.json | 35 -- ...data_google.spanner.admin.database.v1.json | 2 +- ...data_google.spanner.admin.instance.v1.json | 2 +- .../snippet_metadata_google.spanner.v1.json | 2 +- 23 files changed, 810 insertions(+), 235 deletions(-) delete mode 100644 .github/.OwlBot.lock.yaml delete mode 100644 .github/.OwlBot.yaml delete mode 100644 .github/auto-approve.yml delete mode 100644 .github/release-please.yml delete mode 100644 .github/release-trigger.yml delete mode 100644 .github/snippet-bot.yml delete mode 100644 .github/sync-repo-settings.yaml create mode 100644 .librarian/generator-input/.repo-metadata.json rename owlbot.py => .librarian/generator-input/librarian.py (73%) create mode 100644 .librarian/generator-input/noxfile.py create mode 100644 .librarian/generator-input/setup.py create mode 100644 .librarian/state.yaml delete mode 100644 release-please-config.json diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml deleted file mode 100644 index 0ba6990347..0000000000 --- a/.github/.OwlBot.lock.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:fbbc8db67afd8b7d71bf694c5081a32da0c528eba166fbcffb3b6e56ddf907d5 -# created: 2025-10-30T00:16:55.473963098Z diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml deleted file mode 100644 index 5db16e2a9d..0000000000 --- a/.github/.OwlBot.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - -deep-remove-regex: - - /owl-bot-staging - -deep-copy-regex: - - source: /google/spanner/(v.*)/.*-py/(.*) - dest: /owl-bot-staging/spanner/$1/$2 - - source: /google/spanner/admin/instance/(v.*)/.*-py/(.*) - dest: /owl-bot-staging/spanner_admin_instance/$1/$2 - - source: /google/spanner/admin/database/(v.*)/.*-py/(.*) - dest: /owl-bot-staging/spanner_admin_database/$1/$2 - -begin-after-commit-hash: b154da710c5c9eedee127c07f74b6158c9c22382 - diff --git a/.github/auto-approve.yml b/.github/auto-approve.yml deleted file mode 100644 index 311ebbb853..0000000000 --- a/.github/auto-approve.yml +++ /dev/null @@ -1,3 +0,0 @@ -# https://github.com/googleapis/repo-automation-bots/tree/main/packages/auto-approve -processes: - - "OwlBotTemplateChanges" diff --git a/.github/release-please.yml b/.github/release-please.yml deleted file mode 100644 index dbd2cc9deb..0000000000 --- a/.github/release-please.yml +++ /dev/null @@ -1,15 +0,0 @@ -releaseType: python -handleGHRelease: true -manifest: true -# NOTE: this section is generated by synthtool.languages.python -# See https://github.com/googleapis/synthtool/blob/master/synthtool/languages/python.py -branches: -- branch: v2 - handleGHRelease: true - releaseType: python -- branch: v1 - handleGHRelease: true - releaseType: python -- branch: v0 - handleGHRelease: true - releaseType: python diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml deleted file mode 100644 index 3c0f1bfc7e..0000000000 --- a/.github/release-trigger.yml +++ /dev/null @@ -1,2 +0,0 @@ -enabled: true -multiScmName: python-spanner diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml deleted file mode 100644 index d726d1193d..0000000000 --- a/.github/sync-repo-settings.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings -# Rules for main branch protection -branchProtectionRules: -# Identifies the protection rule pattern. Name of the branch to be protected. -# Defaults to `main` -- pattern: main - requiresCodeOwnerReviews: true - requiresStrictStatusChecks: true - requiredStatusCheckContexts: - - 'Kokoro' - - 'Kokoro system-3.12' - - 'cla/google' - - 'Samples - Lint' - - 'Samples - Python 3.9' - - 'Samples - Python 3.12' diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index 67db6136d1..6e5f1af29b 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -17,7 +17,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: 3.13 + python-version: 3.14 - name: Install nox run: python -m pip install nox - name: Check formatting diff --git a/.librarian/generator-input/.repo-metadata.json b/.librarian/generator-input/.repo-metadata.json new file mode 100644 index 0000000000..9569af6e31 --- /dev/null +++ b/.librarian/generator-input/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "name": "spanner", + "name_pretty": "Cloud Spanner", + "product_documentation": "https://cloud.google.com/spanner/docs/", + "client_documentation": "https://cloud.google.com/python/docs/reference/spanner/latest", + "issue_tracker": "https://issuetracker.google.com/issues?q=componentid:190851%2B%20status:open", + "release_level": "stable", + "language": "python", + "library_type": "GAPIC_COMBO", + "repo": "googleapis/python-spanner", + "distribution_name": "google-cloud-spanner", + "api_id": "spanner.googleapis.com", + "requires_billing": true, + "default_version": "v1", + "codeowner_team": "@googleapis/spanner-client-libraries-python", + "api_shortname": "spanner", + "api_description": "is a fully managed, mission-critical, \nrelational database service that offers transactional consistency at global scale, \nschemas, SQL (ANSI 2011 with extensions), and automatic, synchronous replication \nfor high availability.\n\nBe sure to activate the Cloud Spanner API on the Developer's Console to\nuse Cloud Spanner from your project." +} diff --git a/owlbot.py b/.librarian/generator-input/librarian.py similarity index 73% rename from owlbot.py rename to .librarian/generator-input/librarian.py index 4547c4d2d0..31efb36c92 100644 --- a/owlbot.py +++ b/.librarian/generator-input/librarian.py @@ -25,58 +25,9 @@ common = gcp.CommonTemplates() - -def get_staging_dirs( - # This is a customized version of the s.get_staging_dirs() function - # from synthtool to # cater for copying 3 different folders from - # googleapis-gen: - # spanner, spanner/admin/instance and spanner/admin/database. - # Source: - # https://github.com/googleapis/synthtool/blob/master/synthtool/transforms.py#L280 - default_version: Optional[str] = None, - sub_directory: Optional[str] = None, -) -> List[Path]: - """Returns the list of directories, one per version, copied from - https://github.com/googleapis/googleapis-gen. Will return in lexical sorting - order with the exception of the default_version which will be last (if specified). - - Args: - default_version (str): the default version of the API. The directory for this version - will be the last item in the returned list if specified. - sub_directory (str): if a `sub_directory` is provided, only the directories within the - specified `sub_directory` will be returned. - - Returns: the empty list if no file were copied. - """ - - staging = Path("owl-bot-staging") - - if sub_directory: - staging /= sub_directory - - if staging.is_dir(): - # Collect the subdirectories of the staging directory. - versions = [v.name for v in staging.iterdir() if v.is_dir()] - # Reorder the versions so the default version always comes last. - versions = [v for v in versions if v != default_version] - versions.sort() - if default_version is not None: - versions += [default_version] - dirs = [staging / v for v in versions] - for dir in dirs: - s._tracked_paths.add(dir) - return dirs - else: - return [] - - -spanner_default_version = "v1" -spanner_admin_instance_default_version = "v1" -spanner_admin_database_default_version = "v1" - clean_up_generated_samples = True -for library in get_staging_dirs(spanner_default_version, "spanner"): +for library in s.get_staging_dirs("v1"): if clean_up_generated_samples: shutil.rmtree("samples/generated_samples", ignore_errors=True) clean_up_generated_samples = False @@ -202,22 +153,6 @@ def get_staging_dirs( if count < 1: raise Exception("Expected replacements for gRPC channel options not made.") - s.move( - library, - excludes=[ - "google/cloud/spanner/**", - "*.*", - "noxfile.py", - "docs/index.rst", - "google/cloud/spanner_v1/__init__.py", - "**/gapic_version.py", - "testing/constraints-3.7.txt", - ], - ) - -for library in get_staging_dirs( - spanner_admin_instance_default_version, "spanner_admin_instance" -): count = s.replace( [ library / "google/cloud/spanner_admin_instance_v1/services/*/transports/grpc*", @@ -233,14 +168,7 @@ def get_staging_dirs( ) if count < 1: raise Exception("Expected replacements for gRPC channel options not made.") - s.move( - library, - excludes=["google/cloud/spanner_admin_instance/**", "*.*", "docs/index.rst", "noxfile.py", "**/gapic_version.py", "testing/constraints-3.7.txt",], - ) -for library in get_staging_dirs( - spanner_admin_database_default_version, "spanner_admin_database" -): count = s.replace( [ library / "google/cloud/spanner_admin_database_v1/services/*/transports/grpc*", @@ -258,7 +186,16 @@ def get_staging_dirs( raise Exception("Expected replacements for gRPC channel options not made.") s.move( library, - excludes=["google/cloud/spanner_admin_database/**", "*.*", "docs/index.rst", "noxfile.py", "**/gapic_version.py", "testing/constraints-3.7.txt",], + excludes=[ + "google/cloud/spanner/**", + "*.*", + "noxfile.py", + "docs/index.rst", + "google/cloud/spanner_v1/__init__.py", + "testing/constraints-3.7.txt", + "google/cloud/spanner_admin_instance/**", + "google/cloud/spanner_admin_database/**" + ], ) s.remove_staging_dirs() @@ -278,27 +215,12 @@ def get_staging_dirs( templated_files, excludes=[ ".coveragerc", - ".github/workflows", # exclude gh actions as credentials are needed for tests + ".github/**", + ".kokoro/**", "README.rst", - ".github/release-please.yml", - ".kokoro/test-samples-impl.sh", - ".kokoro/presubmit/presubmit.cfg", - ".kokoro/samples/python3.7/**", - ".kokoro/samples/python3.8/**", ], ) -# Ensure CI runs on a new instance each time -s.replace( - ".kokoro/build.sh", - "# Setup project id.", - """\ -# Set up creating a new instance for each system test run -export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=true - -# Setup project id.""", -) - # Update samples folder in CONTRIBUTING.rst s.replace("CONTRIBUTING.rst", "samples/snippets", "samples/samples") @@ -321,4 +243,4 @@ def get_staging_dirs( # Use a python runtime which is available in the owlbot post processor here # https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile -s.shell.run(["nox", "-s", "blacken-3.10"], hide_output=False) +s.shell.run(["nox", "-s", "blacken-3.14"], hide_output=False) diff --git a/.librarian/generator-input/noxfile.py b/.librarian/generator-input/noxfile.py new file mode 100644 index 0000000000..81c522d0d5 --- /dev/null +++ b/.librarian/generator-input/noxfile.py @@ -0,0 +1,595 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! + +from __future__ import absolute_import + +import os +import pathlib +import re +import shutil +from typing import Dict, List +import warnings + +import nox + +FLAKE8_VERSION = "flake8==6.1.0" +BLACK_VERSION = "black[jupyter]==23.7.0" +ISORT_VERSION = "isort==5.11.0" +LINT_PATHS = ["google", "tests", "noxfile.py", "setup.py"] + +DEFAULT_PYTHON_VERSION = "3.14" + +DEFAULT_MOCK_SERVER_TESTS_PYTHON_VERSION = "3.12" +SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.12"] + +UNIT_TEST_PYTHON_VERSIONS: List[str] = [ + "3.9", + "3.10", + "3.11", + "3.12", + "3.13", +] +UNIT_TEST_STANDARD_DEPENDENCIES = [ + "mock", + "asyncmock", + "pytest", + "pytest-cov", + "pytest-asyncio", +] +MOCK_SERVER_ADDITIONAL_DEPENDENCIES = [ + "google-cloud-testutils", +] +UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] +UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [] +UNIT_TEST_DEPENDENCIES: List[str] = [] +UNIT_TEST_EXTRAS: List[str] = [] +UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} + +SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ + "mock", + "pytest", + "google-cloud-testutils", +] +SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] +SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] +SYSTEM_TEST_DEPENDENCIES: List[str] = [] +SYSTEM_TEST_EXTRAS: List[str] = [ + "tracing", +] +SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} + +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +nox.options.sessions = [ + "unit-3.9", + "unit-3.10", + "unit-3.11", + "unit-3.12", + "unit-3.13", + "system", + "cover", + "lint", + "lint_setup_py", + "blacken", + "docs", + "docfx", + "format", +] + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install(FLAKE8_VERSION, BLACK_VERSION) + session.run( + "black", + "--check", + *LINT_PATHS, + ) + session.run("flake8", "google", "tests") + + +# Use a python runtime which is available in the owlbot post processor here +# https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile +@nox.session(python=DEFAULT_PYTHON_VERSION) +def blacken(session): + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run( + "isort", + "--fss", + *LINT_PATHS, + ) + session.run( + "black", + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + session.install("docutils", "pygments", "setuptools>=79.0.1") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + + +def install_unittest_dependencies(session, *constraints): + standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES + session.install(*standard_deps, *constraints) + + if UNIT_TEST_EXTERNAL_DEPENDENCIES: + warnings.warn( + "'unit_test_external_dependencies' is deprecated. Instead, please " + "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.", + DeprecationWarning, + ) + session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_LOCAL_DEPENDENCIES: + session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_EXTRAS_BY_PYTHON: + extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif UNIT_TEST_EXTRAS: + extras = UNIT_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + # XXX Work around Kokoro image's older pip, which borks the OT install. + session.run("pip", "install", "--upgrade", "pip") + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + session.install("-e", ".[tracing]", "-c", constraints_path) + # XXX: Dump installed versions to debug OT issue + session.run("pip", "list") + + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@nox.parametrize( + "protobuf_implementation", + ["python", "upb", "cpp"], +) +def unit(session, protobuf_implementation): + # Install all test dependencies, then install this package in-place. + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): + session.skip("cpp implementation is not supported in python 3.11+") + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + install_unittest_dependencies(session, "-c", constraints_path) + + # TODO(https://github.com/googleapis/synthtool/issues/1976): + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf<4. + if protobuf_implementation == "cpp": + session.install("protobuf<4") + + # Run py.test against the unit tests. + session.run( + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=google", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + + +@nox.session(python=DEFAULT_MOCK_SERVER_TESTS_PYTHON_VERSION) +def mockserver(session): + # Install all test dependencies, then install this package in-place. + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + standard_deps = ( + UNIT_TEST_STANDARD_DEPENDENCIES + + UNIT_TEST_DEPENDENCIES + + MOCK_SERVER_ADDITIONAL_DEPENDENCIES + ) + session.install(*standard_deps, "-c", constraints_path) + session.install("-e", ".", "-c", constraints_path) + + # Run py.test against the mockserver tests. + session.run( + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=google", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "mockserver_tests"), + *session.posargs, + ) + + +def install_systemtest_dependencies(session, *constraints): + # Use pre-release gRPC for system tests. + # Exclude version 1.52.0rc1 which has a known issue. + # See https://github.com/grpc/grpc/issues/32163 + session.install("--pre", "grpcio!=1.52.0rc1") + + session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTERNAL_DEPENDENCIES: + session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_LOCAL_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTRAS_BY_PYTHON: + extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif SYSTEM_TEST_EXTRAS: + extras = SYSTEM_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@nox.parametrize( + "protobuf_implementation,database_dialect", + [ + ("python", "GOOGLE_STANDARD_SQL"), + ("python", "POSTGRESQL"), + ("upb", "GOOGLE_STANDARD_SQL"), + ("upb", "POSTGRESQL"), + ("cpp", "GOOGLE_STANDARD_SQL"), + ("cpp", "POSTGRESQL"), + ], +) +def system(session, protobuf_implementation, database_dialect): + """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Sanity check: Only run tests if the environment variable is set. + if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "") and not os.environ.get( + "SPANNER_EMULATOR_HOST", "" + ): + session.skip( + "Credentials or emulator host must be set via environment variable" + ) + if not ( + os.environ.get("SPANNER_EMULATOR_HOST") or protobuf_implementation == "python" + ): + session.skip( + "Only run system tests on real Spanner with one protobuf implementation to speed up the build" + ) + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): + session.skip("cpp implementation is not supported in python 3.11+") + + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") + + install_systemtest_dependencies(session, "-c", constraints_path) + + # TODO(https://github.com/googleapis/synthtool/issues/1976): + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf<4. + if protobuf_implementation == "cpp": + session.install("protobuf<4") + + # Run py.test against the system tests. + if system_test_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + "SPANNER_DATABASE_DIALECT": database_dialect, + "SKIP_BACKUP_TESTS": "true", + }, + ) + elif system_test_folder_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + "SPANNER_DATABASE_DIALECT": database_dialect, + "SKIP_BACKUP_TESTS": "true", + }, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session): + """Run the final coverage report. + + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=98") + + session.run("coverage", "erase") + + +@nox.session(python="3.10") +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".[tracing]") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "sphinx==4.5.0", + "alabaster", + "recommonmark", + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python="3.10") +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".[tracing]") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "gcp-sphinx-docfx-yaml", + "alabaster", + "recommonmark", + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python="3.13") +@nox.parametrize( + "protobuf_implementation,database_dialect", + [ + ("python", "GOOGLE_STANDARD_SQL"), + ("python", "POSTGRESQL"), + ("upb", "GOOGLE_STANDARD_SQL"), + ("upb", "POSTGRESQL"), + ("cpp", "GOOGLE_STANDARD_SQL"), + ("cpp", "POSTGRESQL"), + ], +) +def prerelease_deps(session, protobuf_implementation, database_dialect): + """Run all tests with prerelease versions of dependencies installed.""" + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): + session.skip("cpp implementation is not supported in python 3.11+") + + # Install all dependencies + session.install("-e", ".[all, tests, tracing]") + unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES + session.install(*unit_deps_all) + system_deps_all = ( + SYSTEM_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_EXTERNAL_DEPENDENCIES + ) + session.install(*system_deps_all) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + constraints_deps = [ + match.group(1) + for match in re.finditer( + r"^\s*([a-zA-Z0-9._-]+)", constraints_text, flags=re.MULTILINE + ) + ] + + if constraints_deps: + session.install(*constraints_deps) + + prerel_deps = [ + "protobuf", + # dependency of grpc + "six", + "grpc-google-iam-v1", + "googleapis-common-protos", + "grpcio", + "grpcio-status", + "google-api-core", + "google-auth", + "proto-plus", + "google-cloud-testutils", + # dependencies of google-cloud-testutils" + "click", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + ] + session.install(*other_deps) + + # Print out prerelease package versions + session.run( + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" + ) + session.run("python", "-c", "import grpc; print(grpc.__version__)") + session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + + session.run( + "py.test", + "tests/unit", + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + "SPANNER_DATABASE_DIALECT": database_dialect, + "SKIP_BACKUP_TESTS": "true", + }, + ) + + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Only run system tests for one protobuf implementation on real Spanner to speed up the build. + if os.environ.get("SPANNER_EMULATOR_HOST") or protobuf_implementation == "python": + # Only run system tests if found. + if os.path.exists(system_test_path): + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + "SPANNER_DATABASE_DIALECT": database_dialect, + "SKIP_BACKUP_TESTS": "true", + }, + ) + elif os.path.exists(system_test_folder_path): + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + "SPANNER_DATABASE_DIALECT": database_dialect, + "SKIP_BACKUP_TESTS": "true", + }, + ) diff --git a/.librarian/generator-input/setup.py b/.librarian/generator-input/setup.py new file mode 100644 index 0000000000..858982f783 --- /dev/null +++ b/.librarian/generator-input/setup.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +import os + +import setuptools # type: ignore + +package_root = os.path.abspath(os.path.dirname(__file__)) + +name = "google-cloud-spanner" + + +description = "Google Cloud Spanner API client library" + +version = {} +with open(os.path.join(package_root, "google/cloud/spanner_v1/gapic_version.py")) as fp: + exec(fp.read(), version) +version = version["__version__"] + +if version[0] == "0": + release_status = "Development Status :: 4 - Beta" +else: + release_status = "Development Status :: 5 - Production/Stable" + +dependencies = [ + "google-api-core[grpc] >= 1.34.0, <3.0.0,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", + "google-cloud-core >= 1.4.4, < 3.0.0", + "grpc-google-iam-v1 >= 0.12.4, <1.0.0", + "proto-plus >= 1.22.0, <2.0.0", + "sqlparse >= 0.4.4", + "proto-plus >= 1.22.2, <2.0.0; python_version>='3.11'", + "protobuf>=3.20.2,<7.0.0,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", + "grpc-interceptor >= 0.15.4", +] +extras = { + "tracing": [ + "opentelemetry-api >= 1.22.0", + "opentelemetry-sdk >= 1.22.0", + "opentelemetry-semantic-conventions >= 0.43b0", + "opentelemetry-resourcedetector-gcp >= 1.8.0a0", + "google-cloud-monitoring >= 2.16.0", + "mmh3 >= 4.1.0 ", + ], + "libcst": "libcst >= 0.2.5", +} + +url = "https://github.com/googleapis/python-spanner" + +package_root = os.path.abspath(os.path.dirname(__file__)) + +readme_filename = os.path.join(package_root, "README.rst") +with io.open(readme_filename, encoding="utf-8") as readme_file: + readme = readme_file.read() + +packages = [ + package + for package in setuptools.find_namespace_packages() + if package.startswith("google") +] + +setuptools.setup( + name=name, + version=version, + description=description, + long_description=readme, + author="Google LLC", + author_email="googleapis-packages@google.com", + license="Apache 2.0", + url=url, + classifiers=[ + release_status, + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Internet", + ], + platforms="Posix; MacOS X; Windows", + packages=packages, + install_requires=dependencies, + extras_require=extras, + python_requires=">=3.9", + include_package_data=True, + zip_safe=False, +) diff --git a/.librarian/state.yaml b/.librarian/state.yaml new file mode 100644 index 0000000000..08fd9350c2 --- /dev/null +++ b/.librarian/state.yaml @@ -0,0 +1,54 @@ +image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:8e2c32496077054105bd06c54a59d6a6694287bc053588e24debe6da6920ad91 +libraries: + - id: google-cloud-spanner + version: 3.59.0 + last_generated_commit: a17b84add8318f780fcc8a027815d5fee644b9f7 + apis: + - path: google/spanner/admin/instance/v1 + service_config: spanner.yaml + - path: google/spanner/admin/database/v1 + service_config: spanner.yaml + - path: google/spanner/v1 + service_config: spanner.yaml + source_roots: + - . + preserve_regex: [] + remove_regex: + - ^google/cloud/spanner_v1/gapic_metadata.json + - ^google/cloud/spanner_v1/gapic_version.py + - ^google/cloud/spanner_v1/py.typed + - ^google/cloud/spanner_v1/services + - ^google/cloud/spanner_v1/types + - ^google/cloud/spanner_admin_database_v1 + - ^google/cloud/spanner_admin_instance_v1 + - ^tests/unit/gapic + - ^tests/__init__.py + - ^tests/unit/__init__.py + - ^.pre-commit-config.yaml + - ^.repo-metadata.json + - ^.trampolinerc + - ^LICENSE + - ^SECURITY.md + - ^mypy.ini + - ^noxfile.py + - ^renovate.json + - ^samples/AUTHORING_GUIDE.md + - ^samples/CONTRIBUTING.md + - ^samples/generated_samples + - ^scripts/fixup_ + - ^setup.py + - ^testing/constraints-3.8 + - ^testing/constraints-3.9 + - ^testing/constraints-3.1 + - ^docs/conf.py + - ^docs/_static + - ^docs/spanner_v1/types_.rst + - ^docs/_templates + - ^docs/spanner_v1/services_.rst + - ^docs/spanner_v1/spanner.rst + - ^docs/spanner_v1/types.rst + - ^docs/spanner_admin_database_v1 + - ^docs/spanner_admin_instance_v1 + - ^docs/multiprocessing.rst + - ^docs/summary_overview.md + tag_format: v{version} diff --git a/docs/conf.py b/docs/conf.py index 64058683e8..010a6b6cda 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,7 @@ # limitations under the License. # # -# google-cloud-spanner-admin-database documentation build configuration file +# google-cloud-spanner documentation build configuration file # # This file is execfile()d with the current directory set to its # containing dir. @@ -81,9 +81,9 @@ root_doc = "index" # General information about the project. -project = "google-cloud-spanner-admin-database" -copyright = "2025, Google, LLC" -author = "Google APIs" +project = u"google-cloud-spanner" +copyright = u"2025, Google, LLC" +author = u"Google APIs" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -155,7 +155,7 @@ # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - "description": "Google Cloud Client Libraries for google-cloud-spanner-admin-database", + "description": "Google Cloud Client Libraries for google-cloud-spanner", "github_user": "googleapis", "github_repo": "google-cloud-python", "github_banner": True, @@ -249,7 +249,7 @@ # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = "google-cloud-spanner-admin-database-doc" +htmlhelp_basename = "google-cloud-spanner-doc" # -- Options for warnings ------------------------------------------------------ @@ -282,8 +282,8 @@ latex_documents = [ ( root_doc, - "google-cloud-spanner-admin-database.tex", - "google-cloud-spanner-admin-database Documentation", + "google-cloud-spanner.tex", + u"google-cloud-spanner Documentation", author, "manual", ) @@ -317,8 +317,8 @@ man_pages = [ ( root_doc, - "google-cloud-spanner-admin-database", - "google-cloud-spanner-admin-database Documentation", + "google-cloud-spanner", + "google-cloud-spanner Documentation", [author], 1, ) @@ -336,11 +336,11 @@ texinfo_documents = [ ( root_doc, - "google-cloud-spanner-admin-database", - "google-cloud-spanner-admin-database Documentation", + "google-cloud-spanner", + "google-cloud-spanner Documentation", author, - "google-cloud-spanner-admin-database", - "google-cloud-spanner-admin-database Library", + "google-cloud-spanner", + "google-cloud-spanner Library", "APIs", ) ] diff --git a/google/cloud/spanner_admin_database_v1/gapic_version.py b/google/cloud/spanner_admin_database_v1/gapic_version.py index 17acb3026a..745f02e051 100644 --- a/google/cloud/spanner_admin_database_v1/gapic_version.py +++ b/google/cloud/spanner_admin_database_v1/gapic_version.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/google/cloud/spanner_admin_instance_v1/gapic_version.py b/google/cloud/spanner_admin_instance_v1/gapic_version.py index 17acb3026a..745f02e051 100644 --- a/google/cloud/spanner_admin_instance_v1/gapic_version.py +++ b/google/cloud/spanner_admin_instance_v1/gapic_version.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/google/cloud/spanner_dbapi/version.py b/google/cloud/spanner_dbapi/version.py index 6fbb80eb90..0ae3005c43 100644 --- a/google/cloud/spanner_dbapi/version.py +++ b/google/cloud/spanner_dbapi/version.py @@ -13,8 +13,8 @@ # limitations under the License. import platform -from google.cloud.spanner_v1 import gapic_version as package_version PY_VERSION = platform.python_version() -VERSION = package_version.__version__ +__version__ = "3.59.0" +VERSION = __version__ DEFAULT_USER_AGENT = "gl-dbapi/" + VERSION diff --git a/google/cloud/spanner_v1/gapic_version.py b/google/cloud/spanner_v1/gapic_version.py index 17acb3026a..745f02e051 100644 --- a/google/cloud/spanner_v1/gapic_version.py +++ b/google/cloud/spanner_v1/gapic_version.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/noxfile.py b/noxfile.py index b101f46b2e..81c522d0d5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -30,9 +30,9 @@ FLAKE8_VERSION = "flake8==6.1.0" BLACK_VERSION = "black[jupyter]==23.7.0" ISORT_VERSION = "isort==5.11.0" -LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] +LINT_PATHS = ["google", "tests", "noxfile.py", "setup.py"] -DEFAULT_PYTHON_VERSION = "3.12" +DEFAULT_PYTHON_VERSION = "3.14" DEFAULT_MOCK_SERVER_TESTS_PYTHON_VERSION = "3.12" SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.12"] @@ -113,7 +113,7 @@ def lint(session): # Use a python runtime which is available in the owlbot post processor here # https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile -@nox.session(python=["3.10", DEFAULT_PYTHON_VERSION]) +@nox.session(python=DEFAULT_PYTHON_VERSION) def blacken(session): """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) diff --git a/release-please-config.json b/release-please-config.json deleted file mode 100644 index faae5c405c..0000000000 --- a/release-please-config.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", - "packages": { - ".": { - "release-type": "python", - "extra-files": [ - "google/cloud/spanner_admin_instance_v1/gapic_version.py", - "google/cloud/spanner_v1/gapic_version.py", - "google/cloud/spanner_admin_database_v1/gapic_version.py", - { - "type": "json", - "path": "samples/generated_samples/snippet_metadata_google.spanner.v1.json", - "jsonpath": "$.clientLibrary.version" - }, - { - "type": "json", - "path": "samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json", - "jsonpath": "$.clientLibrary.version" - }, - { - "type": "json", - "path": "samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json", - "jsonpath": "$.clientLibrary.version" - } - ] - } - }, - "release-type": "python", - "plugins": [ - { - "type": "sentence-case" - } - ], - "initial-version": "0.1.0" -} diff --git a/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json b/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json index e89008727d..e6eeb1f977 100644 --- a/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json +++ b/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-spanner-admin-database", - "version": "0.0.0" + "version": "3.59.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json b/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json index f58e9794e2..92ae0279ef 100644 --- a/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json +++ b/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-spanner-admin-instance", - "version": "0.0.0" + "version": "3.59.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.spanner.v1.json b/samples/generated_samples/snippet_metadata_google.spanner.v1.json index f7f33c3d29..4d84b1ab9a 100644 --- a/samples/generated_samples/snippet_metadata_google.spanner.v1.json +++ b/samples/generated_samples/snippet_metadata_google.spanner.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-spanner", - "version": "0.0.0" + "version": "3.59.0" }, "snippets": [ { From 64aebe7e3ecfec756435f7d102b36f5a41f7cc52 Mon Sep 17 00:00:00 2001 From: Subham Sinha <35077434+sinhasubham@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:27:30 +0530 Subject: [PATCH 358/364] feat(spanner): make built-in metrics enabled by default (#1459) Make built-in metrics enabled by default This change inverts the logic for enabling built-in OpenTelemetry metrics. Previously, metrics were disabled by default and could be enabled by setting `ENABLE_SPANNER_METRICS_ENV_VAR=true`. With this update, metrics are now enabled by default to provide better out-of-the-box observability for users. To disable metrics, users must now set the new environment variable: `SPANNER_DISABLE_BUILTIN_METRICS=true` The old `ENABLE_SPANNER_METRICS_ENV_VAR` is no longer used. Unit tests have been updated to reflect this new opt-out behavior. **BREAKING CHANGE**: Built-in metrics are now enabled by default. Users who previously did not set any environment variables will have metrics collection and export turned on automatically after upgrading. To restore the previous behavior and disable metrics, thry have to set the `SPANNER_DISABLE_BUILTIN_METRICS` environment variable to `true`. --- google/cloud/spanner_v1/client.py | 5 +- google/cloud/spanner_v1/metrics/constants.py | 1 - tests/system/test_metrics.py | 92 ++++++++++++++++++++ tests/unit/test_client.py | 6 +- 4 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 tests/system/test_metrics.py diff --git a/google/cloud/spanner_v1/client.py b/google/cloud/spanner_v1/client.py index eb5b0a6ca6..4d562d354b 100644 --- a/google/cloud/spanner_v1/client.py +++ b/google/cloud/spanner_v1/client.py @@ -52,7 +52,6 @@ from google.cloud.spanner_v1._helpers import _metadata_with_prefix from google.cloud.spanner_v1.instance import Instance from google.cloud.spanner_v1.metrics.constants import ( - ENABLE_SPANNER_METRICS_ENV_VAR, METRIC_EXPORT_INTERVAL_MS, ) from google.cloud.spanner_v1.metrics.spanner_metrics_tracer_factory import ( @@ -75,7 +74,7 @@ _CLIENT_INFO = client_info.ClientInfo(client_library_version=__version__) EMULATOR_ENV_VAR = "SPANNER_EMULATOR_HOST" -ENABLE_BUILTIN_METRICS_ENV_VAR = "SPANNER_ENABLE_BUILTIN_METRICS" +SPANNER_DISABLE_BUILTIN_METRICS_ENV_VAR = "SPANNER_DISABLE_BUILTIN_METRICS" _EMULATOR_HOST_HTTP_SCHEME = ( "%s contains a http scheme. When used with a scheme it may cause gRPC's " "DNS resolver to endlessly attempt to resolve. %s is intended to be used " @@ -102,7 +101,7 @@ def _get_spanner_optimizer_statistics_package(): def _get_spanner_enable_builtin_metrics(): - return os.getenv(ENABLE_SPANNER_METRICS_ENV_VAR) == "true" + return os.getenv(SPANNER_DISABLE_BUILTIN_METRICS_ENV_VAR) != "true" class Client(ClientWithProject): diff --git a/google/cloud/spanner_v1/metrics/constants.py b/google/cloud/spanner_v1/metrics/constants.py index a47aecc9ed..a5f709881b 100644 --- a/google/cloud/spanner_v1/metrics/constants.py +++ b/google/cloud/spanner_v1/metrics/constants.py @@ -20,7 +20,6 @@ GOOGLE_CLOUD_REGION_KEY = "cloud.region" GOOGLE_CLOUD_REGION_GLOBAL = "global" SPANNER_METHOD_PREFIX = "/google.spanner.v1." -ENABLE_SPANNER_METRICS_ENV_VAR = "SPANNER_ENABLE_BUILTIN_METRICS" # Monitored resource labels MONITORED_RES_LABEL_KEY_PROJECT = "project_id" diff --git a/tests/system/test_metrics.py b/tests/system/test_metrics.py new file mode 100644 index 0000000000..acc8d45cee --- /dev/null +++ b/tests/system/test_metrics.py @@ -0,0 +1,92 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import mock +import pytest + +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader + +from google.cloud.spanner_v1 import Client + +# System tests are skipped if the environment variables are not set. +PROJECT = os.environ.get("GOOGLE_CLOUD_PROJECT") +INSTANCE_ID = os.environ.get("SPANNER_TEST_INSTANCE") +DATABASE_ID = "test_metrics_db_system" + + +pytestmark = pytest.mark.skipif( + not all([PROJECT, INSTANCE_ID]), reason="System test environment variables not set." +) + + +@pytest.fixture(scope="module") +def metrics_database(): + """Create a database for the test.""" + client = Client(project=PROJECT) + instance = client.instance(INSTANCE_ID) + database = instance.database(DATABASE_ID) + if database.exists(): # Clean up from previous failed run + database.drop() + op = database.create() + op.result(timeout=300) # Wait for creation to complete + yield database + if database.exists(): + database.drop() + + +def test_builtin_metrics_with_default_otel(metrics_database): + """ + Verifies that built-in metrics are collected by default when a + transaction is executed. + """ + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + # Patch the client's metric setup to use our in-memory reader. + with mock.patch( + "google.cloud.spanner_v1.client.MeterProvider", + return_value=meter_provider, + ): + with mock.patch.dict(os.environ, {"SPANNER_DISABLE_BUILTIN_METRICS": "false"}): + with metrics_database.snapshot() as snapshot: + list(snapshot.execute_sql("SELECT 1")) + + metric_data = reader.get_metrics_data() + + assert len(metric_data.resource_metrics) >= 1 + assert len(metric_data.resource_metrics[0].scope_metrics) >= 1 + + collected_metrics = { + metric.name + for metric in metric_data.resource_metrics[0].scope_metrics[0].metrics + } + expected_metrics = { + "spanner/operation_latencies", + "spanner/attempt_latencies", + "spanner/operation_count", + "spanner/attempt_count", + "spanner/gfe_latencies", + } + assert expected_metrics.issubset(collected_metrics) + + for metric in metric_data.resource_metrics[0].scope_metrics[0].metrics: + if metric.name == "spanner/operation_count": + point = next(iter(metric.data.data_points)) + assert point.value == 1 + assert point.attributes["method"] == "ExecuteSql" + return + + pytest.fail("Metric 'spanner/operation_count' not found.") diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index f0d246673a..94481836ce 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -22,6 +22,7 @@ from tests._builders import build_scoped_credentials +@mock.patch.dict(os.environ, {"SPANNER_DISABLE_BUILTIN_METRICS": "true"}) class TestClient(unittest.TestCase): PROJECT = "PROJECT" PATH = "projects/%s" % (PROJECT,) @@ -161,8 +162,7 @@ def test_constructor_custom_client_info(self): creds = build_scoped_credentials() self._constructor_test_helper(expected_scopes, creds, client_info=client_info) - # Disable metrics to avoid google.auth.default calls from Metric Exporter - @mock.patch.dict(os.environ, {"SPANNER_ENABLE_BUILTIN_METRICS": ""}) + # Metrics are disabled by default for tests in this class def test_constructor_implicit_credentials(self): from google.cloud.spanner_v1 import client as MUT @@ -255,8 +255,8 @@ def test_constructor_w_directed_read_options(self): expected_scopes, creds, directed_read_options=self.DIRECTED_READ_OPTIONS ) - @mock.patch.dict(os.environ, {"SPANNER_ENABLE_BUILTIN_METRICS": "true"}) @mock.patch("google.cloud.spanner_v1.client.SpannerMetricsTracerFactory") + @mock.patch.dict(os.environ, {"SPANNER_DISABLE_BUILTIN_METRICS": "false"}) def test_constructor_w_metrics_initialization_error( self, mock_spanner_metrics_factory ): From f1ebc43ba4c1ee3a8ee77ae4b0b2468937f06b71 Mon Sep 17 00:00:00 2001 From: surbhigarg92 Date: Tue, 9 Dec 2025 16:18:16 +0530 Subject: [PATCH 359/364] fix: Provide Spanner Option to disable metrics (#1460) --- google/cloud/spanner_v1/client.py | 10 ++++++++-- tests/unit/test_client.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/google/cloud/spanner_v1/client.py b/google/cloud/spanner_v1/client.py index 4d562d354b..5f72905616 100644 --- a/google/cloud/spanner_v1/client.py +++ b/google/cloud/spanner_v1/client.py @@ -100,7 +100,7 @@ def _get_spanner_optimizer_statistics_package(): log = logging.getLogger(__name__) -def _get_spanner_enable_builtin_metrics(): +def _get_spanner_enable_builtin_metrics_env(): return os.getenv(SPANNER_DISABLE_BUILTIN_METRICS_ENV_VAR) != "true" @@ -180,6 +180,10 @@ class Client(ClientWithProject): This is intended only for experimental host spanner endpoints. If set, this will override the `api_endpoint` in `client_options`. + :type disable_builtin_metrics: bool + :param disable_builtin_metrics: (Optional) Default False. Set to True to disable + the Spanner built-in metrics collection and exporting. + :raises: :class:`ValueError ` if both ``read_only`` and ``admin`` are :data:`True` """ @@ -205,6 +209,7 @@ def __init__( observability_options=None, default_transaction_options: Optional[DefaultTransactionOptions] = None, experimental_host=None, + disable_builtin_metrics=False, ): self._emulator_host = _get_spanner_emulator_host() self._experimental_host = experimental_host @@ -248,7 +253,8 @@ def __init__( warnings.warn(_EMULATOR_HOST_HTTP_SCHEME) # Check flag to enable Spanner builtin metrics if ( - _get_spanner_enable_builtin_metrics() + _get_spanner_enable_builtin_metrics_env() + and not disable_builtin_metrics and HAS_GOOGLE_CLOUD_MONITORING_INSTALLED ): meter_provider = metrics.NoOpMeterProvider() diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 94481836ce..ab00d45268 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -278,6 +278,37 @@ def test_constructor_w_metrics_initialization_error( ) mock_spanner_metrics_factory.assert_called_once() + @mock.patch("google.cloud.spanner_v1.client.SpannerMetricsTracerFactory") + @mock.patch.dict(os.environ, {"SPANNER_DISABLE_BUILTIN_METRICS": "true"}) + def test_constructor_w_disable_builtin_metrics_using_env( + self, mock_spanner_metrics_factory + ): + """ + Test that Client constructor disable metrics using Spanner Option. + """ + from google.cloud.spanner_v1.client import Client + + creds = build_scoped_credentials() + client = Client(project=self.PROJECT, credentials=creds) + self.assertIsNotNone(client) + mock_spanner_metrics_factory.assert_called_once_with(enabled=False) + + @mock.patch("google.cloud.spanner_v1.client.SpannerMetricsTracerFactory") + def test_constructor_w_disable_builtin_metrics_using_option( + self, mock_spanner_metrics_factory + ): + """ + Test that Client constructor disable metrics using Spanner Option. + """ + from google.cloud.spanner_v1.client import Client + + creds = build_scoped_credentials() + client = Client( + project=self.PROJECT, credentials=creds, disable_builtin_metrics=True + ) + self.assertIsNotNone(client) + mock_spanner_metrics_factory.assert_called_once_with(enabled=False) + def test_constructor_route_to_leader_disbled(self): from google.cloud.spanner_v1 import client as MUT From 7e30073838d75ecb04884e0f209f5de52d897873 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 9 Dec 2025 19:03:15 -0500 Subject: [PATCH 360/364] chore(python): Add support for Python 3.14 (#1456) This PR adds support for Python 3.14 to the library. Key changes include: Key changes include: - Updates to `owlbot.py` to include Python 3.14. - Adding Python 3.14 to the test matrix in `.github/workflows/presubmit.yaml`. - Verified `setup.py` includes the Python 3.14 classifier. - Verified `CONTRIBUTING.rst` includes Python 3.14. - Verified `noxfile.py` updates for 3.14. - Updated Kokoro configurations in `.kokoro/presubmit/` to use `system-3.14` session. - Updated `librarian.py` to account for post-processing. - Fixed a concurrency issue in `tests/unit/test_spanner.py` to make the test suite pass on Python 3.14. Towards internal issue: b/375664027 --------- Co-authored-by: Owl Bot Co-authored-by: gcf-owl-bot[bot] <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: Subham Sinha <35077434+sinhasubham@users.noreply.github.com> Co-authored-by: surbhigarg92 --- ...against-emulator-with-regular-session.yaml | 2 +- .../integration-tests-against-emulator.yaml | 2 +- .github/workflows/mock_server_tests.yaml | 2 +- .github/workflows/presubmit.yaml | 2 +- .../integration-regular-sessions-enabled.cfg | 2 +- .../{system-3.12.cfg => system-3.14.cfg} | 4 +- .librarian/generator-input/librarian.py | 9 +- CONTRIBUTING.rst | 2 + noxfile.py | 28 ++++- setup.py | 1 + testing/constraints-3.14.txt | 2 +- tests/unit/test_spanner.py | 116 ++++++++---------- 12 files changed, 90 insertions(+), 82 deletions(-) rename .kokoro/presubmit/{system-3.12.cfg => system-3.14.cfg} (81%) diff --git a/.github/workflows/integration-tests-against-emulator-with-regular-session.yaml b/.github/workflows/integration-tests-against-emulator-with-regular-session.yaml index 826a3b7629..3f2d3b7ba2 100644 --- a/.github/workflows/integration-tests-against-emulator-with-regular-session.yaml +++ b/.github/workflows/integration-tests-against-emulator-with-regular-session.yaml @@ -21,7 +21,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: 3.13 + python-version: 3.14 - name: Install nox run: python -m pip install nox - name: Run system tests diff --git a/.github/workflows/integration-tests-against-emulator.yaml b/.github/workflows/integration-tests-against-emulator.yaml index e7158307b8..e8078107bc 100644 --- a/.github/workflows/integration-tests-against-emulator.yaml +++ b/.github/workflows/integration-tests-against-emulator.yaml @@ -21,7 +21,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: 3.13 + python-version: 3.14 - name: Install nox run: python -m pip install nox - name: Run system tests diff --git a/.github/workflows/mock_server_tests.yaml b/.github/workflows/mock_server_tests.yaml index b705c98191..d16feac517 100644 --- a/.github/workflows/mock_server_tests.yaml +++ b/.github/workflows/mock_server_tests.yaml @@ -14,7 +14,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: 3.13 + python-version: 3.14 - name: Install nox run: python -m pip install nox - name: Run mock server tests diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index 6e5f1af29b..56386a746c 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -27,7 +27,7 @@ jobs: strategy: fail-fast: false matrix: - python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout code diff --git a/.kokoro/presubmit/integration-regular-sessions-enabled.cfg b/.kokoro/presubmit/integration-regular-sessions-enabled.cfg index 1f646bebf2..439abd4ba5 100644 --- a/.kokoro/presubmit/integration-regular-sessions-enabled.cfg +++ b/.kokoro/presubmit/integration-regular-sessions-enabled.cfg @@ -3,7 +3,7 @@ # Only run a subset of all nox sessions env_vars: { key: "NOX_SESSION" - value: "unit-3.9 unit-3.12 system-3.12" + value: "unit-3.9 unit-3.14 system-3.14" } env_vars: { diff --git a/.kokoro/presubmit/system-3.12.cfg b/.kokoro/presubmit/system-3.14.cfg similarity index 81% rename from .kokoro/presubmit/system-3.12.cfg rename to .kokoro/presubmit/system-3.14.cfg index 78cdc5e851..73904141ba 100644 --- a/.kokoro/presubmit/system-3.12.cfg +++ b/.kokoro/presubmit/system-3.14.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "system-3.12" -} \ No newline at end of file + value: "system-3.14" +} diff --git a/.librarian/generator-input/librarian.py b/.librarian/generator-input/librarian.py index 31efb36c92..46c2e8dbb4 100644 --- a/.librarian/generator-input/librarian.py +++ b/.librarian/generator-input/librarian.py @@ -209,7 +209,8 @@ cov_level=98, split_system_tests=True, system_test_extras=["tracing"], - system_test_python_versions=["3.12"] + system_test_python_versions=["3.12"], + unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] ) s.move( templated_files, @@ -224,6 +225,12 @@ # Update samples folder in CONTRIBUTING.rst s.replace("CONTRIBUTING.rst", "samples/snippets", "samples/samples") +s.replace( + "noxfile.py", + '''session.python in \("3.11", "3.12", "3.13"\)''', + '''session.python in ("3.11", "3.12", "3.13", "3.14")''' +) + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 76e9061cd2..60be7c4f93 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -226,12 +226,14 @@ We support: - `Python 3.11`_ - `Python 3.12`_ - `Python 3.13`_ +- `Python 3.14`_ .. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ .. _Python 3.11: https://docs.python.org/3.11/ .. _Python 3.12: https://docs.python.org/3.12/ .. _Python 3.13: https://docs.python.org/3.13/ +.. _Python 3.14: https://docs.python.org/3.14/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/noxfile.py b/noxfile.py index 81c522d0d5..82715de072 100644 --- a/noxfile.py +++ b/noxfile.py @@ -35,7 +35,7 @@ DEFAULT_PYTHON_VERSION = "3.14" DEFAULT_MOCK_SERVER_TESTS_PYTHON_VERSION = "3.12" -SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.12"] +SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.14"] UNIT_TEST_PYTHON_VERSIONS: List[str] = [ "3.9", @@ -43,6 +43,7 @@ "3.11", "3.12", "3.13", + "3.14", ] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", @@ -81,6 +82,7 @@ "unit-3.11", "unit-3.12", "unit-3.13", + "unit-3.14", "system", "cover", "lint", @@ -195,7 +197,12 @@ def install_unittest_dependencies(session, *constraints): def unit(session, protobuf_implementation): # Install all test dependencies, then install this package in-place. - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): + if protobuf_implementation == "cpp" and session.python in ( + "3.11", + "3.12", + "3.13", + "3.14", + ): session.skip("cpp implementation is not supported in python 3.11+") constraints_path = str( @@ -213,6 +220,7 @@ def unit(session, protobuf_implementation): session.run( "py.test", "--quiet", + "-s", f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=google", "--cov=tests/unit", @@ -326,7 +334,12 @@ def system(session, protobuf_implementation, database_dialect): "Only run system tests on real Spanner with one protobuf implementation to speed up the build" ) - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): + if protobuf_implementation == "cpp" and session.python in ( + "3.11", + "3.12", + "3.13", + "3.14", + ): session.skip("cpp implementation is not supported in python 3.11+") # Install pyopenssl for mTLS testing. @@ -470,7 +483,7 @@ def docfx(session): ) -@nox.session(python="3.13") +@nox.session(python="3.14") @nox.parametrize( "protobuf_implementation,database_dialect", [ @@ -485,7 +498,12 @@ def docfx(session): def prerelease_deps(session, protobuf_implementation, database_dialect): """Run all tests with prerelease versions of dependencies installed.""" - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): + if protobuf_implementation == "cpp" and session.python in ( + "3.11", + "3.12", + "3.13", + "3.14", + ): session.skip("cpp implementation is not supported in python 3.11+") # Install all dependencies diff --git a/setup.py b/setup.py index 216b095d0b..fdd911bfd1 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.14", "Operating System :: OS Independent", "Topic :: Internet", ], diff --git a/testing/constraints-3.14.txt b/testing/constraints-3.14.txt index 2ae5a677e8..92054fc895 100644 --- a/testing/constraints-3.14.txt +++ b/testing/constraints-3.14.txt @@ -10,4 +10,4 @@ google-auth>=2 grpcio>=1 proto-plus>=1 protobuf>=6 -grpc-google-iam-v1>=0 +grpc-google-iam-v1>=0 \ No newline at end of file diff --git a/tests/unit/test_spanner.py b/tests/unit/test_spanner.py index e35b817858..d1de23d2d0 100644 --- a/tests/unit/test_spanner.py +++ b/tests/unit/test_spanner.py @@ -475,7 +475,6 @@ def _batch_update_helper( self.assertEqual(status, expected_status) self.assertEqual(row_counts, expected_row_counts) - self.assertEqual(transaction._execute_sql_request_count, count + 1) def _batch_update_expected_request(self, begin=True, count=0): if begin is True: @@ -1071,37 +1070,27 @@ def test_transaction_for_concurrent_statement_should_begin_one_transaction_with_ ) self.assertEqual(api.execute_batch_dml.call_count, 2) - self.assertEqual( - api.execute_batch_dml.call_args_list, - [ - mock.call( - request=self._batch_update_expected_request(), - metadata=[ - ("google-cloud-resource-prefix", database.name), - ("x-goog-spanner-route-to-leader", "true"), - ( - "x-goog-spanner-request-id", - f"1.{REQ_RAND_PROCESS_ID}.{database._nth_client_id}.{database._channel_id}.1.1", - ), - ], - retry=RETRY, - timeout=TIMEOUT, - ), - mock.call( - request=self._batch_update_expected_request(begin=False), - metadata=[ - ("google-cloud-resource-prefix", database.name), - ("x-goog-spanner-route-to-leader", "true"), - ( - "x-goog-spanner-request-id", - f"1.{REQ_RAND_PROCESS_ID}.{database._nth_client_id}.{database._channel_id}.2.1", - ), - ], - retry=RETRY, - timeout=TIMEOUT, - ), - ], + + call_args_list = api.execute_batch_dml.call_args_list + + request_ids = [] + for call in call_args_list: + metadata = call.kwargs["metadata"] + self.assertEqual(len(metadata), 3) + self.assertEqual( + metadata[0], ("google-cloud-resource-prefix", database.name) + ) + self.assertEqual(metadata[1], ("x-goog-spanner-route-to-leader", "true")) + self.assertEqual(metadata[2][0], "x-goog-spanner-request-id") + request_ids.append(metadata[2][1]) + self.assertEqual(call.kwargs["retry"], RETRY) + self.assertEqual(call.kwargs["timeout"], TIMEOUT) + + expected_id_suffixes = ["1.1", "2.1"] + actual_id_suffixes = sorted( + [".".join(rid.split(".")[-2:]) for rid in request_ids] ) + self.assertEqual(actual_id_suffixes, expected_id_suffixes) def test_transaction_for_concurrent_statement_should_begin_one_transaction_with_read( self, @@ -1131,11 +1120,6 @@ def test_transaction_for_concurrent_statement_should_begin_one_transaction_with_ self._execute_update_helper(transaction=transaction, api=api) - begin_read_write_count = sum( - [1 for call in api.mock_calls if "read_write" in call.kwargs.__str__()] - ) - - self.assertEqual(begin_read_write_count, 1) api.execute_sql.assert_any_call( request=self._execute_update_expected_request(database, begin=False), retry=RETRY, @@ -1150,41 +1134,37 @@ def test_transaction_for_concurrent_statement_should_begin_one_transaction_with_ ], ) - self.assertEqual( - api.streaming_read.call_args_list, - [ - mock.call( - request=self._read_helper_expected_request(), - metadata=[ - ("google-cloud-resource-prefix", database.name), - ("x-goog-spanner-route-to-leader", "true"), - ( - "x-goog-spanner-request-id", - f"1.{REQ_RAND_PROCESS_ID}.{database._nth_client_id}.{database._channel_id}.1.1", - ), - ], - retry=RETRY, - timeout=TIMEOUT, - ), - mock.call( - request=self._read_helper_expected_request(begin=False), - metadata=[ - ("google-cloud-resource-prefix", database.name), - ("x-goog-spanner-route-to-leader", "true"), - ( - "x-goog-spanner-request-id", - f"1.{REQ_RAND_PROCESS_ID}.{database._nth_client_id}.{database._channel_id}.2.1", - ), - ], - retry=RETRY, - timeout=TIMEOUT, - ), - ], - ) - self.assertEqual(api.execute_sql.call_count, 1) self.assertEqual(api.streaming_read.call_count, 2) + call_args_list = api.streaming_read.call_args_list + + expected_requests = [ + self._read_helper_expected_request(), + self._read_helper_expected_request(begin=False), + ] + actual_requests = [call.kwargs["request"] for call in call_args_list] + self.assertCountEqual(actual_requests, expected_requests) + + request_ids = [] + for call in call_args_list: + metadata = call.kwargs["metadata"] + self.assertEqual(len(metadata), 3) + self.assertEqual( + metadata[0], ("google-cloud-resource-prefix", database.name) + ) + self.assertEqual(metadata[1], ("x-goog-spanner-route-to-leader", "true")) + self.assertEqual(metadata[2][0], "x-goog-spanner-request-id") + request_ids.append(metadata[2][1]) + self.assertEqual(call.kwargs["retry"], RETRY) + self.assertEqual(call.kwargs["timeout"], TIMEOUT) + + expected_id_suffixes = ["1.1", "2.1"] + actual_id_suffixes = sorted( + [".".join(rid.split(".")[-2:]) for rid in request_ids] + ) + self.assertEqual(actual_id_suffixes, expected_id_suffixes) + def test_transaction_for_concurrent_statement_should_begin_one_transaction_with_query( self, ): From 24394d691f36b4d43f6f60c0bf4b983dd0a241c2 Mon Sep 17 00:00:00 2001 From: rahul2393 Date: Wed, 10 Dec 2025 22:44:22 +0530 Subject: [PATCH 361/364] chore: librarian release pull request: 20251210T220651Z (#1462) PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. Librarian Version: v1.0.0 Language Image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:8e2c32496077054105bd06c54a59d6a6694287bc053588e24debe6da6920ad91
google-cloud-spanner: 3.60.0 ## [3.60.0](https://togithub.com/googleapis/python-spanner/compare/v3.59.0...v3.60.0) (2025-12-10) ### Features * make built-in metrics enabled by default (#1459) ([64aebe7e](https://togithub.com/googleapis/python-spanner/commit/64aebe7e)) * Exposing AutoscalingConfig in InstancePartition ([8b6f1540](https://togithub.com/googleapis/python-spanner/commit/8b6f1540)) * add support for experimental host (#1452) ([9535e5e0](https://togithub.com/googleapis/python-spanner/commit/9535e5e0)) * enable OpenTelemetry metrics and tracing by default (#1410) ([bb5095df](https://togithub.com/googleapis/python-spanner/commit/bb5095df)) * add cloud.region, request_tag and transaction_tag in span attributes (#1449) ([d37fb80a](https://togithub.com/googleapis/python-spanner/commit/d37fb80a)) * Add QueryAdvisorResult for query plan (PiperOrigin-RevId: 832425466) ([e08260fe](https://togithub.com/googleapis/python-spanner/commit/e08260fe)) * Add Send and Ack mutations for Queues (PiperOrigin-RevId: 832425466) ([e08260fe](https://togithub.com/googleapis/python-spanner/commit/e08260fe)) * Add Spanner location API (#1457) (PiperOrigin-RevId: 833474957) ([e08260fe](https://togithub.com/googleapis/python-spanner/commit/e08260fe)) ### Bug Fixes * Deprecate credentials_file argument ([8b6f1540](https://togithub.com/googleapis/python-spanner/commit/8b6f1540)) * configure keepAlive time for gRPC TCP connections (#1448) ([efb2833e](https://togithub.com/googleapis/python-spanner/commit/efb2833e)) * Provide Spanner Option to disable metrics (#1460) ([f1ebc43b](https://togithub.com/googleapis/python-spanner/commit/f1ebc43b)) ### Documentation * Update description for the BatchCreateSessionsRequest and Session (PiperOrigin-RevId: 832425466) ([e08260fe](https://togithub.com/googleapis/python-spanner/commit/e08260fe)) * Update description for the IsolationLevel (PiperOrigin-RevId: 832425466) ([e08260fe](https://togithub.com/googleapis/python-spanner/commit/e08260fe))
--- .librarian/state.yaml | 2 +- CHANGELOG.md | 27 +++++++++++++++++++ .../gapic_version.py | 2 +- .../gapic_version.py | 2 +- google/cloud/spanner_dbapi/version.py | 2 +- google/cloud/spanner_v1/gapic_version.py | 2 +- ...data_google.spanner.admin.database.v1.json | 2 +- ...data_google.spanner.admin.instance.v1.json | 2 +- .../snippet_metadata_google.spanner.v1.json | 2 +- 9 files changed, 35 insertions(+), 8 deletions(-) diff --git a/.librarian/state.yaml b/.librarian/state.yaml index 08fd9350c2..2b8a475a0a 100644 --- a/.librarian/state.yaml +++ b/.librarian/state.yaml @@ -1,7 +1,7 @@ image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:8e2c32496077054105bd06c54a59d6a6694287bc053588e24debe6da6920ad91 libraries: - id: google-cloud-spanner - version: 3.59.0 + version: 3.60.0 last_generated_commit: a17b84add8318f780fcc8a027815d5fee644b9f7 apis: - path: google/spanner/admin/instance/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index b5bbe07325..0a5a487e85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,33 @@ [1]: https://pypi.org/project/google-cloud-spanner/#history +## [3.60.0](https://github.com/googleapis/python-spanner/compare/v3.59.0...v3.60.0) (2025-12-10) + + +### Documentation + +* Update description for the BatchCreateSessionsRequest and Session ([e08260fe24b62313d7964572eeb963eb8c3c923f](https://github.com/googleapis/python-spanner/commit/e08260fe24b62313d7964572eeb963eb8c3c923f)) +* Update description for the IsolationLevel ([e08260fe24b62313d7964572eeb963eb8c3c923f](https://github.com/googleapis/python-spanner/commit/e08260fe24b62313d7964572eeb963eb8c3c923f)) + + +### Features + +* make built-in metrics enabled by default (#1459) ([64aebe7e3ecfec756435f7d102b36f5a41f7cc52](https://github.com/googleapis/python-spanner/commit/64aebe7e3ecfec756435f7d102b36f5a41f7cc52)) +* Add Spanner location API (#1457) ([e08260fe24b62313d7964572eeb963eb8c3c923f](https://github.com/googleapis/python-spanner/commit/e08260fe24b62313d7964572eeb963eb8c3c923f)) +* Add Send and Ack mutations for Queues ([e08260fe24b62313d7964572eeb963eb8c3c923f](https://github.com/googleapis/python-spanner/commit/e08260fe24b62313d7964572eeb963eb8c3c923f)) +* Add QueryAdvisorResult for query plan ([e08260fe24b62313d7964572eeb963eb8c3c923f](https://github.com/googleapis/python-spanner/commit/e08260fe24b62313d7964572eeb963eb8c3c923f)) +* add cloud.region, request_tag and transaction_tag in span attributes (#1449) ([d37fb80a39aea859059ae7d85adc75095a6e14e6](https://github.com/googleapis/python-spanner/commit/d37fb80a39aea859059ae7d85adc75095a6e14e6)) +* Exposing AutoscalingConfig in InstancePartition ([8b6f154085543953556acde161a739414988b7f0](https://github.com/googleapis/python-spanner/commit/8b6f154085543953556acde161a739414988b7f0)) +* enable OpenTelemetry metrics and tracing by default (#1410) ([bb5095dfb615159a575933a332382ba93ba4bbd1](https://github.com/googleapis/python-spanner/commit/bb5095dfb615159a575933a332382ba93ba4bbd1)) +* add support for experimental host (#1452) ([9535e5e096f6ab53f2817af4fd7ac1fa2ca71660](https://github.com/googleapis/python-spanner/commit/9535e5e096f6ab53f2817af4fd7ac1fa2ca71660)) + + +### Bug Fixes + +* Provide Spanner Option to disable metrics (#1460) ([f1ebc43ba4c1ee3a8ee77ae4b0b2468937f06b71](https://github.com/googleapis/python-spanner/commit/f1ebc43ba4c1ee3a8ee77ae4b0b2468937f06b71)) +* Deprecate credentials_file argument ([8b6f154085543953556acde161a739414988b7f0](https://github.com/googleapis/python-spanner/commit/8b6f154085543953556acde161a739414988b7f0)) +* configure keepAlive time for gRPC TCP connections (#1448) ([efb2833e52e54b096e552a4d91f94b017ac733bb](https://github.com/googleapis/python-spanner/commit/efb2833e52e54b096e552a4d91f94b017ac733bb)) + ## [3.59.0](https://github.com/googleapis/python-spanner/compare/v3.58.0...v3.59.0) (2025-10-18) diff --git a/google/cloud/spanner_admin_database_v1/gapic_version.py b/google/cloud/spanner_admin_database_v1/gapic_version.py index 745f02e051..992322a033 100644 --- a/google/cloud/spanner_admin_database_v1/gapic_version.py +++ b/google/cloud/spanner_admin_database_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "3.59.0" # {x-release-please-version} +__version__ = "3.60.0" # {x-release-please-version} diff --git a/google/cloud/spanner_admin_instance_v1/gapic_version.py b/google/cloud/spanner_admin_instance_v1/gapic_version.py index 745f02e051..992322a033 100644 --- a/google/cloud/spanner_admin_instance_v1/gapic_version.py +++ b/google/cloud/spanner_admin_instance_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "3.59.0" # {x-release-please-version} +__version__ = "3.60.0" # {x-release-please-version} diff --git a/google/cloud/spanner_dbapi/version.py b/google/cloud/spanner_dbapi/version.py index 0ae3005c43..ee7431572b 100644 --- a/google/cloud/spanner_dbapi/version.py +++ b/google/cloud/spanner_dbapi/version.py @@ -15,6 +15,6 @@ import platform PY_VERSION = platform.python_version() -__version__ = "3.59.0" +__version__ = "3.60.0" VERSION = __version__ DEFAULT_USER_AGENT = "gl-dbapi/" + VERSION diff --git a/google/cloud/spanner_v1/gapic_version.py b/google/cloud/spanner_v1/gapic_version.py index 745f02e051..992322a033 100644 --- a/google/cloud/spanner_v1/gapic_version.py +++ b/google/cloud/spanner_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "3.59.0" # {x-release-please-version} +__version__ = "3.60.0" # {x-release-please-version} diff --git a/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json b/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json index e6eeb1f977..0bfe97d988 100644 --- a/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json +++ b/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-spanner-admin-database", - "version": "3.59.0" + "version": "3.60.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json b/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json index 92ae0279ef..9b51de3471 100644 --- a/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json +++ b/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-spanner-admin-instance", - "version": "3.59.0" + "version": "3.60.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.spanner.v1.json b/samples/generated_samples/snippet_metadata_google.spanner.v1.json index 4d84b1ab9a..1ec5a82e5a 100644 --- a/samples/generated_samples/snippet_metadata_google.spanner.v1.json +++ b/samples/generated_samples/snippet_metadata_google.spanner.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-spanner", - "version": "3.59.0" + "version": "3.60.0" }, "snippets": [ { From df87c3ed55db7cffa2eed4d7316ca5c375af1c5a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 16 Dec 2025 12:53:43 -0800 Subject: [PATCH 362/364] feat(gapic): support mTLS certificates when available (#1467) feat: update image to us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:b8058df4c45e9a6e07f6b4d65b458d0d059241dd34c814f151c8bf6b89211209 --- .librarian/generator-input/noxfile.py | 28 +- .librarian/generator-input/setup.py | 20 +- .librarian/state.yaml | 2 +- .../spanner_admin_database_v1/__init__.py | 104 +++++++ .../services/database_admin/client.py | 46 +++- .../spanner_admin_instance_v1/__init__.py | 104 +++++++ .../services/instance_admin/client.py | 46 +++- .../spanner_v1/services/spanner/client.py | 46 +++- noxfile.py | 4 + ...ixup_spanner_admin_database_v1_keywords.py | 202 -------------- ...ixup_spanner_admin_instance_v1_keywords.py | 196 -------------- scripts/fixup_spanner_v1_keywords.py | 191 ------------- setup.py | 4 + testing/constraints-3.10.txt | 1 - testing/constraints-3.11.txt | 1 - testing/constraints-3.12.txt | 1 - testing/constraints-3.13.txt | 1 - testing/constraints-3.14.txt | 1 - testing/constraints-3.8.txt | 1 - testing/constraints-3.9.txt | 1 - .../test_database_admin.py | 255 ++++++++++++++++-- .../test_instance_admin.py | 255 ++++++++++++++++-- tests/unit/gapic/spanner_v1/test_spanner.py | 255 ++++++++++++++++-- 23 files changed, 1023 insertions(+), 742 deletions(-) delete mode 100644 scripts/fixup_spanner_admin_database_v1_keywords.py delete mode 100644 scripts/fixup_spanner_admin_instance_v1_keywords.py delete mode 100644 scripts/fixup_spanner_v1_keywords.py diff --git a/.librarian/generator-input/noxfile.py b/.librarian/generator-input/noxfile.py index 81c522d0d5..82715de072 100644 --- a/.librarian/generator-input/noxfile.py +++ b/.librarian/generator-input/noxfile.py @@ -35,7 +35,7 @@ DEFAULT_PYTHON_VERSION = "3.14" DEFAULT_MOCK_SERVER_TESTS_PYTHON_VERSION = "3.12" -SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.12"] +SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ["3.14"] UNIT_TEST_PYTHON_VERSIONS: List[str] = [ "3.9", @@ -43,6 +43,7 @@ "3.11", "3.12", "3.13", + "3.14", ] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", @@ -81,6 +82,7 @@ "unit-3.11", "unit-3.12", "unit-3.13", + "unit-3.14", "system", "cover", "lint", @@ -195,7 +197,12 @@ def install_unittest_dependencies(session, *constraints): def unit(session, protobuf_implementation): # Install all test dependencies, then install this package in-place. - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): + if protobuf_implementation == "cpp" and session.python in ( + "3.11", + "3.12", + "3.13", + "3.14", + ): session.skip("cpp implementation is not supported in python 3.11+") constraints_path = str( @@ -213,6 +220,7 @@ def unit(session, protobuf_implementation): session.run( "py.test", "--quiet", + "-s", f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=google", "--cov=tests/unit", @@ -326,7 +334,12 @@ def system(session, protobuf_implementation, database_dialect): "Only run system tests on real Spanner with one protobuf implementation to speed up the build" ) - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): + if protobuf_implementation == "cpp" and session.python in ( + "3.11", + "3.12", + "3.13", + "3.14", + ): session.skip("cpp implementation is not supported in python 3.11+") # Install pyopenssl for mTLS testing. @@ -470,7 +483,7 @@ def docfx(session): ) -@nox.session(python="3.13") +@nox.session(python="3.14") @nox.parametrize( "protobuf_implementation,database_dialect", [ @@ -485,7 +498,12 @@ def docfx(session): def prerelease_deps(session, protobuf_implementation, database_dialect): """Run all tests with prerelease versions of dependencies installed.""" - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): + if protobuf_implementation == "cpp" and session.python in ( + "3.11", + "3.12", + "3.13", + "3.14", + ): session.skip("cpp implementation is not supported in python 3.11+") # Install all dependencies diff --git a/.librarian/generator-input/setup.py b/.librarian/generator-input/setup.py index 858982f783..fdd911bfd1 100644 --- a/.librarian/generator-input/setup.py +++ b/.librarian/generator-input/setup.py @@ -44,18 +44,15 @@ "proto-plus >= 1.22.2, <2.0.0; python_version>='3.11'", "protobuf>=3.20.2,<7.0.0,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", "grpc-interceptor >= 0.15.4", + # Make OpenTelemetry a core dependency + "opentelemetry-api >= 1.22.0", + "opentelemetry-sdk >= 1.22.0", + "opentelemetry-semantic-conventions >= 0.43b0", + "opentelemetry-resourcedetector-gcp >= 1.8.0a0", + "google-cloud-monitoring >= 2.16.0", + "mmh3 >= 4.1.0 ", ] -extras = { - "tracing": [ - "opentelemetry-api >= 1.22.0", - "opentelemetry-sdk >= 1.22.0", - "opentelemetry-semantic-conventions >= 0.43b0", - "opentelemetry-resourcedetector-gcp >= 1.8.0a0", - "google-cloud-monitoring >= 2.16.0", - "mmh3 >= 4.1.0 ", - ], - "libcst": "libcst >= 0.2.5", -} +extras = {"libcst": "libcst >= 0.2.5"} url = "https://github.com/googleapis/python-spanner" @@ -90,6 +87,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.14", "Operating System :: OS Independent", "Topic :: Internet", ], diff --git a/.librarian/state.yaml b/.librarian/state.yaml index 2b8a475a0a..6fb8be06a7 100644 --- a/.librarian/state.yaml +++ b/.librarian/state.yaml @@ -1,4 +1,4 @@ -image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:8e2c32496077054105bd06c54a59d6a6694287bc053588e24debe6da6920ad91 +image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:b8058df4c45e9a6e07f6b4d65b458d0d059241dd34c814f151c8bf6b89211209 libraries: - id: google-cloud-spanner version: 3.60.0 diff --git a/google/cloud/spanner_admin_database_v1/__init__.py b/google/cloud/spanner_admin_database_v1/__init__.py index d7fddf0236..42b15fe254 100644 --- a/google/cloud/spanner_admin_database_v1/__init__.py +++ b/google/cloud/spanner_admin_database_v1/__init__.py @@ -15,8 +15,18 @@ # from google.cloud.spanner_admin_database_v1 import gapic_version as package_version +import google.api_core as api_core +import sys + __version__ = package_version.__version__ +if sys.version_info >= (3, 8): # pragma: NO COVER + from importlib import metadata +else: # pragma: NO COVER + # TODO(https://github.com/googleapis/python-api-core/issues/835): Remove + # this code path once we drop support for Python 3.7 + import importlib_metadata as metadata + from .services.database_admin import DatabaseAdminClient from .services.database_admin import DatabaseAdminAsyncClient @@ -83,6 +93,100 @@ from .types.spanner_database_admin import UpdateDatabaseRequest from .types.spanner_database_admin import RestoreSourceType +if hasattr(api_core, "check_python_version") and hasattr( + api_core, "check_dependency_versions" +): # pragma: NO COVER + api_core.check_python_version("google.cloud.spanner_admin_database_v1") # type: ignore + api_core.check_dependency_versions("google.cloud.spanner_admin_database_v1") # type: ignore +else: # pragma: NO COVER + # An older version of api_core is installed which does not define the + # functions above. We do equivalent checks manually. + try: + import warnings + import sys + + _py_version_str = sys.version.split()[0] + _package_label = "google.cloud.spanner_admin_database_v1" + if sys.version_info < (3, 9): + warnings.warn( + "You are using a non-supported Python version " + + f"({_py_version_str}). Google will not post any further " + + f"updates to {_package_label} supporting this Python version. " + + "Please upgrade to the latest Python version, or at " + + f"least to Python 3.9, and then update {_package_label}.", + FutureWarning, + ) + if sys.version_info[:2] == (3, 9): + warnings.warn( + f"You are using a Python version ({_py_version_str}) " + + f"which Google will stop supporting in {_package_label} in " + + "January 2026. Please " + + "upgrade to the latest Python version, or at " + + "least to Python 3.10, before then, and " + + f"then update {_package_label}.", + FutureWarning, + ) + + def parse_version_to_tuple(version_string: str): + """Safely converts a semantic version string to a comparable tuple of integers. + Example: "4.25.8" -> (4, 25, 8) + Ignores non-numeric parts and handles common version formats. + Args: + version_string: Version string in the format "x.y.z" or "x.y.z" + Returns: + Tuple of integers for the parsed version string. + """ + parts = [] + for part in version_string.split("."): + try: + parts.append(int(part)) + except ValueError: + # If it's a non-numeric part (e.g., '1.0.0b1' -> 'b1'), stop here. + # This is a simplification compared to 'packaging.parse_version', but sufficient + # for comparing strictly numeric semantic versions. + break + return tuple(parts) + + def _get_version(dependency_name): + try: + version_string: str = metadata.version(dependency_name) + parsed_version = parse_version_to_tuple(version_string) + return (parsed_version, version_string) + except Exception: + # Catch exceptions from metadata.version() (e.g., PackageNotFoundError) + # or errors during parse_version_to_tuple + return (None, "--") + + _dependency_package = "google.protobuf" + _next_supported_version = "4.25.8" + _next_supported_version_tuple = (4, 25, 8) + _recommendation = " (we recommend 6.x)" + (_version_used, _version_used_string) = _get_version(_dependency_package) + if _version_used and _version_used < _next_supported_version_tuple: + warnings.warn( + f"Package {_package_label} depends on " + + f"{_dependency_package}, currently installed at version " + + f"{_version_used_string}. Future updates to " + + f"{_package_label} will require {_dependency_package} at " + + f"version {_next_supported_version} or higher{_recommendation}." + + " Please ensure " + + "that either (a) your Python environment doesn't pin the " + + f"version of {_dependency_package}, so that updates to " + + f"{_package_label} can require the higher version, or " + + "(b) you manually update your Python environment to use at " + + f"least version {_next_supported_version} of " + + f"{_dependency_package}.", + FutureWarning, + ) + except Exception: + warnings.warn( + "Could not determine the version of Python " + + "currently being used. To continue receiving " + + "updates for {_package_label}, ensure you are " + + "using a supported version of Python; see " + + "https://devguide.python.org/versions/" + ) + __all__ = ( "DatabaseAdminAsyncClient", "AddSplitPointsRequest", diff --git a/google/cloud/spanner_admin_database_v1/services/database_admin/client.py b/google/cloud/spanner_admin_database_v1/services/database_admin/client.py index 5f85aa39b1..057aa677f8 100644 --- a/google/cloud/spanner_admin_database_v1/services/database_admin/client.py +++ b/google/cloud/spanner_admin_database_v1/services/database_admin/client.py @@ -172,6 +172,34 @@ def _get_default_mtls_endpoint(api_endpoint): _DEFAULT_ENDPOINT_TEMPLATE = "spanner.{UNIVERSE_DOMAIN}" _DEFAULT_UNIVERSE = "googleapis.com" + @staticmethod + def _use_client_cert_effective(): + """Returns whether client certificate should be used for mTLS if the + google-auth version supports should_use_client_cert automatic mTLS enablement. + + Alternatively, read from the GOOGLE_API_USE_CLIENT_CERTIFICATE env var. + + Returns: + bool: whether client certificate should be used for mTLS + Raises: + ValueError: (If using a version of google-auth without should_use_client_cert and + GOOGLE_API_USE_CLIENT_CERTIFICATE is set to an unexpected value.) + """ + # check if google-auth version supports should_use_client_cert for automatic mTLS enablement + if hasattr(mtls, "should_use_client_cert"): # pragma: NO COVER + return mtls.should_use_client_cert() + else: # pragma: NO COVER + # if unsupported, fallback to reading from env var + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + @classmethod def from_service_account_info(cls, info: dict, *args, **kwargs): """Creates an instance of this client using the provided credentials @@ -518,12 +546,8 @@ def get_mtls_endpoint_and_cert_source( ) if client_options is None: client_options = client_options_lib.ClientOptions() - use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_client_cert = DatabaseAdminClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" @@ -531,7 +555,7 @@ def get_mtls_endpoint_and_cert_source( # Figure out the client cert source to use. client_cert_source = None - if use_client_cert == "true": + if use_client_cert: if client_options.client_cert_source: client_cert_source = client_options.client_cert_source elif mtls.has_default_client_cert_source(): @@ -563,20 +587,14 @@ def _read_environment_variables(): google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT is not any of ["auto", "never", "always"]. """ - use_client_cert = os.getenv( - "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" - ).lower() + use_client_cert = DatabaseAdminClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + return use_client_cert, use_mtls_endpoint, universe_domain_env @staticmethod def _get_client_cert_source(provided_cert_source, use_cert_flag): diff --git a/google/cloud/spanner_admin_instance_v1/__init__.py b/google/cloud/spanner_admin_instance_v1/__init__.py index 5368b59895..261949561f 100644 --- a/google/cloud/spanner_admin_instance_v1/__init__.py +++ b/google/cloud/spanner_admin_instance_v1/__init__.py @@ -15,8 +15,18 @@ # from google.cloud.spanner_admin_instance_v1 import gapic_version as package_version +import google.api_core as api_core +import sys + __version__ = package_version.__version__ +if sys.version_info >= (3, 8): # pragma: NO COVER + from importlib import metadata +else: # pragma: NO COVER + # TODO(https://github.com/googleapis/python-api-core/issues/835): Remove + # this code path once we drop support for Python 3.7 + import importlib_metadata as metadata + from .services.instance_admin import InstanceAdminClient from .services.instance_admin import InstanceAdminAsyncClient @@ -63,6 +73,100 @@ from .types.spanner_instance_admin import UpdateInstancePartitionRequest from .types.spanner_instance_admin import UpdateInstanceRequest +if hasattr(api_core, "check_python_version") and hasattr( + api_core, "check_dependency_versions" +): # pragma: NO COVER + api_core.check_python_version("google.cloud.spanner_admin_instance_v1") # type: ignore + api_core.check_dependency_versions("google.cloud.spanner_admin_instance_v1") # type: ignore +else: # pragma: NO COVER + # An older version of api_core is installed which does not define the + # functions above. We do equivalent checks manually. + try: + import warnings + import sys + + _py_version_str = sys.version.split()[0] + _package_label = "google.cloud.spanner_admin_instance_v1" + if sys.version_info < (3, 9): + warnings.warn( + "You are using a non-supported Python version " + + f"({_py_version_str}). Google will not post any further " + + f"updates to {_package_label} supporting this Python version. " + + "Please upgrade to the latest Python version, or at " + + f"least to Python 3.9, and then update {_package_label}.", + FutureWarning, + ) + if sys.version_info[:2] == (3, 9): + warnings.warn( + f"You are using a Python version ({_py_version_str}) " + + f"which Google will stop supporting in {_package_label} in " + + "January 2026. Please " + + "upgrade to the latest Python version, or at " + + "least to Python 3.10, before then, and " + + f"then update {_package_label}.", + FutureWarning, + ) + + def parse_version_to_tuple(version_string: str): + """Safely converts a semantic version string to a comparable tuple of integers. + Example: "4.25.8" -> (4, 25, 8) + Ignores non-numeric parts and handles common version formats. + Args: + version_string: Version string in the format "x.y.z" or "x.y.z" + Returns: + Tuple of integers for the parsed version string. + """ + parts = [] + for part in version_string.split("."): + try: + parts.append(int(part)) + except ValueError: + # If it's a non-numeric part (e.g., '1.0.0b1' -> 'b1'), stop here. + # This is a simplification compared to 'packaging.parse_version', but sufficient + # for comparing strictly numeric semantic versions. + break + return tuple(parts) + + def _get_version(dependency_name): + try: + version_string: str = metadata.version(dependency_name) + parsed_version = parse_version_to_tuple(version_string) + return (parsed_version, version_string) + except Exception: + # Catch exceptions from metadata.version() (e.g., PackageNotFoundError) + # or errors during parse_version_to_tuple + return (None, "--") + + _dependency_package = "google.protobuf" + _next_supported_version = "4.25.8" + _next_supported_version_tuple = (4, 25, 8) + _recommendation = " (we recommend 6.x)" + (_version_used, _version_used_string) = _get_version(_dependency_package) + if _version_used and _version_used < _next_supported_version_tuple: + warnings.warn( + f"Package {_package_label} depends on " + + f"{_dependency_package}, currently installed at version " + + f"{_version_used_string}. Future updates to " + + f"{_package_label} will require {_dependency_package} at " + + f"version {_next_supported_version} or higher{_recommendation}." + + " Please ensure " + + "that either (a) your Python environment doesn't pin the " + + f"version of {_dependency_package}, so that updates to " + + f"{_package_label} can require the higher version, or " + + "(b) you manually update your Python environment to use at " + + f"least version {_next_supported_version} of " + + f"{_dependency_package}.", + FutureWarning, + ) + except Exception: + warnings.warn( + "Could not determine the version of Python " + + "currently being used. To continue receiving " + + "updates for {_package_label}, ensure you are " + + "using a supported version of Python; see " + + "https://devguide.python.org/versions/" + ) + __all__ = ( "InstanceAdminAsyncClient", "AutoscalingConfig", diff --git a/google/cloud/spanner_admin_instance_v1/services/instance_admin/client.py b/google/cloud/spanner_admin_instance_v1/services/instance_admin/client.py index c0fe398c3a..0a2bc9afce 100644 --- a/google/cloud/spanner_admin_instance_v1/services/instance_admin/client.py +++ b/google/cloud/spanner_admin_instance_v1/services/instance_admin/client.py @@ -178,6 +178,34 @@ def _get_default_mtls_endpoint(api_endpoint): _DEFAULT_ENDPOINT_TEMPLATE = "spanner.{UNIVERSE_DOMAIN}" _DEFAULT_UNIVERSE = "googleapis.com" + @staticmethod + def _use_client_cert_effective(): + """Returns whether client certificate should be used for mTLS if the + google-auth version supports should_use_client_cert automatic mTLS enablement. + + Alternatively, read from the GOOGLE_API_USE_CLIENT_CERTIFICATE env var. + + Returns: + bool: whether client certificate should be used for mTLS + Raises: + ValueError: (If using a version of google-auth without should_use_client_cert and + GOOGLE_API_USE_CLIENT_CERTIFICATE is set to an unexpected value.) + """ + # check if google-auth version supports should_use_client_cert for automatic mTLS enablement + if hasattr(mtls, "should_use_client_cert"): # pragma: NO COVER + return mtls.should_use_client_cert() + else: # pragma: NO COVER + # if unsupported, fallback to reading from env var + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + @classmethod def from_service_account_info(cls, info: dict, *args, **kwargs): """Creates an instance of this client using the provided credentials @@ -402,12 +430,8 @@ def get_mtls_endpoint_and_cert_source( ) if client_options is None: client_options = client_options_lib.ClientOptions() - use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_client_cert = InstanceAdminClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" @@ -415,7 +439,7 @@ def get_mtls_endpoint_and_cert_source( # Figure out the client cert source to use. client_cert_source = None - if use_client_cert == "true": + if use_client_cert: if client_options.client_cert_source: client_cert_source = client_options.client_cert_source elif mtls.has_default_client_cert_source(): @@ -447,20 +471,14 @@ def _read_environment_variables(): google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT is not any of ["auto", "never", "always"]. """ - use_client_cert = os.getenv( - "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" - ).lower() + use_client_cert = InstanceAdminClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + return use_client_cert, use_mtls_endpoint, universe_domain_env @staticmethod def _get_client_cert_source(provided_cert_source, use_cert_flag): diff --git a/google/cloud/spanner_v1/services/spanner/client.py b/google/cloud/spanner_v1/services/spanner/client.py index d542dd89ef..8083e74c7c 100644 --- a/google/cloud/spanner_v1/services/spanner/client.py +++ b/google/cloud/spanner_v1/services/spanner/client.py @@ -160,6 +160,34 @@ def _get_default_mtls_endpoint(api_endpoint): _DEFAULT_ENDPOINT_TEMPLATE = "spanner.{UNIVERSE_DOMAIN}" _DEFAULT_UNIVERSE = "googleapis.com" + @staticmethod + def _use_client_cert_effective(): + """Returns whether client certificate should be used for mTLS if the + google-auth version supports should_use_client_cert automatic mTLS enablement. + + Alternatively, read from the GOOGLE_API_USE_CLIENT_CERTIFICATE env var. + + Returns: + bool: whether client certificate should be used for mTLS + Raises: + ValueError: (If using a version of google-auth without should_use_client_cert and + GOOGLE_API_USE_CLIENT_CERTIFICATE is set to an unexpected value.) + """ + # check if google-auth version supports should_use_client_cert for automatic mTLS enablement + if hasattr(mtls, "should_use_client_cert"): # pragma: NO COVER + return mtls.should_use_client_cert() + else: # pragma: NO COVER + # if unsupported, fallback to reading from env var + use_client_cert_str = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + @classmethod def from_service_account_info(cls, info: dict, *args, **kwargs): """Creates an instance of this client using the provided credentials @@ -371,12 +399,8 @@ def get_mtls_endpoint_and_cert_source( ) if client_options is None: client_options = client_options_lib.ClientOptions() - use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_client_cert = SpannerClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" @@ -384,7 +408,7 @@ def get_mtls_endpoint_and_cert_source( # Figure out the client cert source to use. client_cert_source = None - if use_client_cert == "true": + if use_client_cert: if client_options.client_cert_source: client_cert_source = client_options.client_cert_source elif mtls.has_default_client_cert_source(): @@ -416,20 +440,14 @@ def _read_environment_variables(): google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT is not any of ["auto", "never", "always"]. """ - use_client_cert = os.getenv( - "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" - ).lower() + use_client_cert = SpannerClient._use_client_cert_effective() use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") - if use_client_cert not in ("true", "false"): - raise ValueError( - "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) if use_mtls_endpoint not in ("auto", "never", "always"): raise MutualTLSChannelError( "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + return use_client_cert, use_mtls_endpoint, universe_domain_env @staticmethod def _get_client_cert_source(provided_cert_source, use_cert_flag): diff --git a/noxfile.py b/noxfile.py index 82715de072..62d67d0be1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,6 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# DO NOT EDIT THIS FILE OUTSIDE OF `.librarian/generator-input` +# The source of truth for this file is `.librarian/generator-input` + + # Generated by synthtool. DO NOT EDIT! from __future__ import absolute_import diff --git a/scripts/fixup_spanner_admin_database_v1_keywords.py b/scripts/fixup_spanner_admin_database_v1_keywords.py deleted file mode 100644 index d642e9a0e3..0000000000 --- a/scripts/fixup_spanner_admin_database_v1_keywords.py +++ /dev/null @@ -1,202 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -import libcst as cst -import pathlib -import sys -from typing import (Any, Callable, Dict, List, Sequence, Tuple) - - -def partition( - predicate: Callable[[Any], bool], - iterator: Sequence[Any] -) -> Tuple[List[Any], List[Any]]: - """A stable, out-of-place partition.""" - results = ([], []) - - for i in iterator: - results[int(predicate(i))].append(i) - - # Returns trueList, falseList - return results[1], results[0] - - -class spanner_admin_databaseCallTransformer(cst.CSTTransformer): - CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') - METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { - 'add_split_points': ('database', 'split_points', 'initiator', ), - 'copy_backup': ('parent', 'backup_id', 'source_backup', 'expire_time', 'encryption_config', ), - 'create_backup': ('parent', 'backup_id', 'backup', 'encryption_config', ), - 'create_backup_schedule': ('parent', 'backup_schedule_id', 'backup_schedule', ), - 'create_database': ('parent', 'create_statement', 'extra_statements', 'encryption_config', 'database_dialect', 'proto_descriptors', ), - 'delete_backup': ('name', ), - 'delete_backup_schedule': ('name', ), - 'drop_database': ('database', ), - 'get_backup': ('name', ), - 'get_backup_schedule': ('name', ), - 'get_database': ('name', ), - 'get_database_ddl': ('database', ), - 'get_iam_policy': ('resource', 'options', ), - 'internal_update_graph_operation': ('database', 'operation_id', 'vm_identity_token', 'progress', 'status', ), - 'list_backup_operations': ('parent', 'filter', 'page_size', 'page_token', ), - 'list_backups': ('parent', 'filter', 'page_size', 'page_token', ), - 'list_backup_schedules': ('parent', 'page_size', 'page_token', ), - 'list_database_operations': ('parent', 'filter', 'page_size', 'page_token', ), - 'list_database_roles': ('parent', 'page_size', 'page_token', ), - 'list_databases': ('parent', 'page_size', 'page_token', ), - 'restore_database': ('parent', 'database_id', 'backup', 'encryption_config', ), - 'set_iam_policy': ('resource', 'policy', 'update_mask', ), - 'test_iam_permissions': ('resource', 'permissions', ), - 'update_backup': ('backup', 'update_mask', ), - 'update_backup_schedule': ('backup_schedule', 'update_mask', ), - 'update_database': ('database', 'update_mask', ), - 'update_database_ddl': ('database', 'statements', 'operation_id', 'proto_descriptors', 'throughput_mode', ), - } - - def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: - try: - key = original.func.attr.value - kword_params = self.METHOD_TO_PARAMS[key] - except (AttributeError, KeyError): - # Either not a method from the API or too convoluted to be sure. - return updated - - # If the existing code is valid, keyword args come after positional args. - # Therefore, all positional args must map to the first parameters. - args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) - if any(k.keyword.value == "request" for k in kwargs): - # We've already fixed this file, don't fix it again. - return updated - - kwargs, ctrl_kwargs = partition( - lambda a: a.keyword.value not in self.CTRL_PARAMS, - kwargs - ) - - args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] - ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) - for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) - - request_arg = cst.Arg( - value=cst.Dict([ - cst.DictElement( - cst.SimpleString("'{}'".format(name)), -cst.Element(value=arg.value) - ) - # Note: the args + kwargs looks silly, but keep in mind that - # the control parameters had to be stripped out, and that - # those could have been passed positionally or by keyword. - for name, arg in zip(kword_params, args + kwargs)]), - keyword=cst.Name("request") - ) - - return updated.with_changes( - args=[request_arg] + ctrl_kwargs - ) - - -def fix_files( - in_dir: pathlib.Path, - out_dir: pathlib.Path, - *, - transformer=spanner_admin_databaseCallTransformer(), -): - """Duplicate the input dir to the output dir, fixing file method calls. - - Preconditions: - * in_dir is a real directory - * out_dir is a real, empty directory - """ - pyfile_gen = ( - pathlib.Path(os.path.join(root, f)) - for root, _, files in os.walk(in_dir) - for f in files if os.path.splitext(f)[1] == ".py" - ) - - for fpath in pyfile_gen: - with open(fpath, 'r') as f: - src = f.read() - - # Parse the code and insert method call fixes. - tree = cst.parse_module(src) - updated = tree.visit(transformer) - - # Create the path and directory structure for the new file. - updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) - updated_path.parent.mkdir(parents=True, exist_ok=True) - - # Generate the updated source file at the corresponding path. - with open(updated_path, 'w') as f: - f.write(updated.code) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="""Fix up source that uses the spanner_admin_database client library. - -The existing sources are NOT overwritten but are copied to output_dir with changes made. - -Note: This tool operates at a best-effort level at converting positional - parameters in client method calls to keyword based parameters. - Cases where it WILL FAIL include - A) * or ** expansion in a method call. - B) Calls via function or method alias (includes free function calls) - C) Indirect or dispatched calls (e.g. the method is looked up dynamically) - - These all constitute false negatives. The tool will also detect false - positives when an API method shares a name with another method. -""") - parser.add_argument( - '-d', - '--input-directory', - required=True, - dest='input_dir', - help='the input directory to walk for python files to fix up', - ) - parser.add_argument( - '-o', - '--output-directory', - required=True, - dest='output_dir', - help='the directory to output files fixed via un-flattening', - ) - args = parser.parse_args() - input_dir = pathlib.Path(args.input_dir) - output_dir = pathlib.Path(args.output_dir) - if not input_dir.is_dir(): - print( - f"input directory '{input_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if not output_dir.is_dir(): - print( - f"output directory '{output_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if os.listdir(output_dir): - print( - f"output directory '{output_dir}' is not empty", - file=sys.stderr, - ) - sys.exit(-1) - - fix_files(input_dir, output_dir) diff --git a/scripts/fixup_spanner_admin_instance_v1_keywords.py b/scripts/fixup_spanner_admin_instance_v1_keywords.py deleted file mode 100644 index 8200af5099..0000000000 --- a/scripts/fixup_spanner_admin_instance_v1_keywords.py +++ /dev/null @@ -1,196 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -import libcst as cst -import pathlib -import sys -from typing import (Any, Callable, Dict, List, Sequence, Tuple) - - -def partition( - predicate: Callable[[Any], bool], - iterator: Sequence[Any] -) -> Tuple[List[Any], List[Any]]: - """A stable, out-of-place partition.""" - results = ([], []) - - for i in iterator: - results[int(predicate(i))].append(i) - - # Returns trueList, falseList - return results[1], results[0] - - -class spanner_admin_instanceCallTransformer(cst.CSTTransformer): - CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') - METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { - 'create_instance': ('parent', 'instance_id', 'instance', ), - 'create_instance_config': ('parent', 'instance_config_id', 'instance_config', 'validate_only', ), - 'create_instance_partition': ('parent', 'instance_partition_id', 'instance_partition', ), - 'delete_instance': ('name', ), - 'delete_instance_config': ('name', 'etag', 'validate_only', ), - 'delete_instance_partition': ('name', 'etag', ), - 'get_iam_policy': ('resource', 'options', ), - 'get_instance': ('name', 'field_mask', ), - 'get_instance_config': ('name', ), - 'get_instance_partition': ('name', ), - 'list_instance_config_operations': ('parent', 'filter', 'page_size', 'page_token', ), - 'list_instance_configs': ('parent', 'page_size', 'page_token', ), - 'list_instance_partition_operations': ('parent', 'filter', 'page_size', 'page_token', 'instance_partition_deadline', ), - 'list_instance_partitions': ('parent', 'page_size', 'page_token', 'instance_partition_deadline', ), - 'list_instances': ('parent', 'page_size', 'page_token', 'filter', 'instance_deadline', ), - 'move_instance': ('name', 'target_config', ), - 'set_iam_policy': ('resource', 'policy', 'update_mask', ), - 'test_iam_permissions': ('resource', 'permissions', ), - 'update_instance': ('instance', 'field_mask', ), - 'update_instance_config': ('instance_config', 'update_mask', 'validate_only', ), - 'update_instance_partition': ('instance_partition', 'field_mask', ), - } - - def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: - try: - key = original.func.attr.value - kword_params = self.METHOD_TO_PARAMS[key] - except (AttributeError, KeyError): - # Either not a method from the API or too convoluted to be sure. - return updated - - # If the existing code is valid, keyword args come after positional args. - # Therefore, all positional args must map to the first parameters. - args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) - if any(k.keyword.value == "request" for k in kwargs): - # We've already fixed this file, don't fix it again. - return updated - - kwargs, ctrl_kwargs = partition( - lambda a: a.keyword.value not in self.CTRL_PARAMS, - kwargs - ) - - args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] - ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) - for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) - - request_arg = cst.Arg( - value=cst.Dict([ - cst.DictElement( - cst.SimpleString("'{}'".format(name)), -cst.Element(value=arg.value) - ) - # Note: the args + kwargs looks silly, but keep in mind that - # the control parameters had to be stripped out, and that - # those could have been passed positionally or by keyword. - for name, arg in zip(kword_params, args + kwargs)]), - keyword=cst.Name("request") - ) - - return updated.with_changes( - args=[request_arg] + ctrl_kwargs - ) - - -def fix_files( - in_dir: pathlib.Path, - out_dir: pathlib.Path, - *, - transformer=spanner_admin_instanceCallTransformer(), -): - """Duplicate the input dir to the output dir, fixing file method calls. - - Preconditions: - * in_dir is a real directory - * out_dir is a real, empty directory - """ - pyfile_gen = ( - pathlib.Path(os.path.join(root, f)) - for root, _, files in os.walk(in_dir) - for f in files if os.path.splitext(f)[1] == ".py" - ) - - for fpath in pyfile_gen: - with open(fpath, 'r') as f: - src = f.read() - - # Parse the code and insert method call fixes. - tree = cst.parse_module(src) - updated = tree.visit(transformer) - - # Create the path and directory structure for the new file. - updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) - updated_path.parent.mkdir(parents=True, exist_ok=True) - - # Generate the updated source file at the corresponding path. - with open(updated_path, 'w') as f: - f.write(updated.code) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="""Fix up source that uses the spanner_admin_instance client library. - -The existing sources are NOT overwritten but are copied to output_dir with changes made. - -Note: This tool operates at a best-effort level at converting positional - parameters in client method calls to keyword based parameters. - Cases where it WILL FAIL include - A) * or ** expansion in a method call. - B) Calls via function or method alias (includes free function calls) - C) Indirect or dispatched calls (e.g. the method is looked up dynamically) - - These all constitute false negatives. The tool will also detect false - positives when an API method shares a name with another method. -""") - parser.add_argument( - '-d', - '--input-directory', - required=True, - dest='input_dir', - help='the input directory to walk for python files to fix up', - ) - parser.add_argument( - '-o', - '--output-directory', - required=True, - dest='output_dir', - help='the directory to output files fixed via un-flattening', - ) - args = parser.parse_args() - input_dir = pathlib.Path(args.input_dir) - output_dir = pathlib.Path(args.output_dir) - if not input_dir.is_dir(): - print( - f"input directory '{input_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if not output_dir.is_dir(): - print( - f"output directory '{output_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if os.listdir(output_dir): - print( - f"output directory '{output_dir}' is not empty", - file=sys.stderr, - ) - sys.exit(-1) - - fix_files(input_dir, output_dir) diff --git a/scripts/fixup_spanner_v1_keywords.py b/scripts/fixup_spanner_v1_keywords.py deleted file mode 100644 index e0787f13b4..0000000000 --- a/scripts/fixup_spanner_v1_keywords.py +++ /dev/null @@ -1,191 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -import libcst as cst -import pathlib -import sys -from typing import (Any, Callable, Dict, List, Sequence, Tuple) - - -def partition( - predicate: Callable[[Any], bool], - iterator: Sequence[Any] -) -> Tuple[List[Any], List[Any]]: - """A stable, out-of-place partition.""" - results = ([], []) - - for i in iterator: - results[int(predicate(i))].append(i) - - # Returns trueList, falseList - return results[1], results[0] - - -class spannerCallTransformer(cst.CSTTransformer): - CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') - METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { - 'batch_create_sessions': ('database', 'session_count', 'session_template', ), - 'batch_write': ('session', 'mutation_groups', 'request_options', 'exclude_txn_from_change_streams', ), - 'begin_transaction': ('session', 'options', 'request_options', 'mutation_key', ), - 'commit': ('session', 'transaction_id', 'single_use_transaction', 'mutations', 'return_commit_stats', 'max_commit_delay', 'request_options', 'precommit_token', ), - 'create_session': ('database', 'session', ), - 'delete_session': ('name', ), - 'execute_batch_dml': ('session', 'transaction', 'statements', 'seqno', 'request_options', 'last_statements', ), - 'execute_sql': ('session', 'sql', 'transaction', 'params', 'param_types', 'resume_token', 'query_mode', 'partition_token', 'seqno', 'query_options', 'request_options', 'directed_read_options', 'data_boost_enabled', 'last_statement', 'routing_hint', ), - 'execute_streaming_sql': ('session', 'sql', 'transaction', 'params', 'param_types', 'resume_token', 'query_mode', 'partition_token', 'seqno', 'query_options', 'request_options', 'directed_read_options', 'data_boost_enabled', 'last_statement', 'routing_hint', ), - 'get_session': ('name', ), - 'list_sessions': ('database', 'page_size', 'page_token', 'filter', ), - 'partition_query': ('session', 'sql', 'transaction', 'params', 'param_types', 'partition_options', ), - 'partition_read': ('session', 'table', 'key_set', 'transaction', 'index', 'columns', 'partition_options', ), - 'read': ('session', 'table', 'columns', 'key_set', 'transaction', 'index', 'limit', 'resume_token', 'partition_token', 'request_options', 'directed_read_options', 'data_boost_enabled', 'order_by', 'lock_hint', 'routing_hint', ), - 'rollback': ('session', 'transaction_id', ), - 'streaming_read': ('session', 'table', 'columns', 'key_set', 'transaction', 'index', 'limit', 'resume_token', 'partition_token', 'request_options', 'directed_read_options', 'data_boost_enabled', 'order_by', 'lock_hint', 'routing_hint', ), - } - - def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: - try: - key = original.func.attr.value - kword_params = self.METHOD_TO_PARAMS[key] - except (AttributeError, KeyError): - # Either not a method from the API or too convoluted to be sure. - return updated - - # If the existing code is valid, keyword args come after positional args. - # Therefore, all positional args must map to the first parameters. - args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) - if any(k.keyword.value == "request" for k in kwargs): - # We've already fixed this file, don't fix it again. - return updated - - kwargs, ctrl_kwargs = partition( - lambda a: a.keyword.value not in self.CTRL_PARAMS, - kwargs - ) - - args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] - ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) - for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) - - request_arg = cst.Arg( - value=cst.Dict([ - cst.DictElement( - cst.SimpleString("'{}'".format(name)), -cst.Element(value=arg.value) - ) - # Note: the args + kwargs looks silly, but keep in mind that - # the control parameters had to be stripped out, and that - # those could have been passed positionally or by keyword. - for name, arg in zip(kword_params, args + kwargs)]), - keyword=cst.Name("request") - ) - - return updated.with_changes( - args=[request_arg] + ctrl_kwargs - ) - - -def fix_files( - in_dir: pathlib.Path, - out_dir: pathlib.Path, - *, - transformer=spannerCallTransformer(), -): - """Duplicate the input dir to the output dir, fixing file method calls. - - Preconditions: - * in_dir is a real directory - * out_dir is a real, empty directory - """ - pyfile_gen = ( - pathlib.Path(os.path.join(root, f)) - for root, _, files in os.walk(in_dir) - for f in files if os.path.splitext(f)[1] == ".py" - ) - - for fpath in pyfile_gen: - with open(fpath, 'r') as f: - src = f.read() - - # Parse the code and insert method call fixes. - tree = cst.parse_module(src) - updated = tree.visit(transformer) - - # Create the path and directory structure for the new file. - updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) - updated_path.parent.mkdir(parents=True, exist_ok=True) - - # Generate the updated source file at the corresponding path. - with open(updated_path, 'w') as f: - f.write(updated.code) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="""Fix up source that uses the spanner client library. - -The existing sources are NOT overwritten but are copied to output_dir with changes made. - -Note: This tool operates at a best-effort level at converting positional - parameters in client method calls to keyword based parameters. - Cases where it WILL FAIL include - A) * or ** expansion in a method call. - B) Calls via function or method alias (includes free function calls) - C) Indirect or dispatched calls (e.g. the method is looked up dynamically) - - These all constitute false negatives. The tool will also detect false - positives when an API method shares a name with another method. -""") - parser.add_argument( - '-d', - '--input-directory', - required=True, - dest='input_dir', - help='the input directory to walk for python files to fix up', - ) - parser.add_argument( - '-o', - '--output-directory', - required=True, - dest='output_dir', - help='the directory to output files fixed via un-flattening', - ) - args = parser.parse_args() - input_dir = pathlib.Path(args.input_dir) - output_dir = pathlib.Path(args.output_dir) - if not input_dir.is_dir(): - print( - f"input directory '{input_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if not output_dir.is_dir(): - print( - f"output directory '{output_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if os.listdir(output_dir): - print( - f"output directory '{output_dir}' is not empty", - file=sys.stderr, - ) - sys.exit(-1) - - fix_files(input_dir, output_dir) diff --git a/setup.py b/setup.py index fdd911bfd1..5e46a79e96 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +# DO NOT EDIT THIS FILE OUTSIDE OF `.librarian/generator-input` +# The source of truth for this file is `.librarian/generator-input` + import io import os diff --git a/testing/constraints-3.10.txt b/testing/constraints-3.10.txt index ef1c92ffff..93e6826f2a 100644 --- a/testing/constraints-3.10.txt +++ b/testing/constraints-3.10.txt @@ -6,4 +6,3 @@ google-auth grpcio proto-plus protobuf -grpc-google-iam-v1 diff --git a/testing/constraints-3.11.txt b/testing/constraints-3.11.txt index ef1c92ffff..93e6826f2a 100644 --- a/testing/constraints-3.11.txt +++ b/testing/constraints-3.11.txt @@ -6,4 +6,3 @@ google-auth grpcio proto-plus protobuf -grpc-google-iam-v1 diff --git a/testing/constraints-3.12.txt b/testing/constraints-3.12.txt index ef1c92ffff..93e6826f2a 100644 --- a/testing/constraints-3.12.txt +++ b/testing/constraints-3.12.txt @@ -6,4 +6,3 @@ google-auth grpcio proto-plus protobuf -grpc-google-iam-v1 diff --git a/testing/constraints-3.13.txt b/testing/constraints-3.13.txt index 2ae5a677e8..1e93c60e50 100644 --- a/testing/constraints-3.13.txt +++ b/testing/constraints-3.13.txt @@ -10,4 +10,3 @@ google-auth>=2 grpcio>=1 proto-plus>=1 protobuf>=6 -grpc-google-iam-v1>=0 diff --git a/testing/constraints-3.14.txt b/testing/constraints-3.14.txt index 92054fc895..1e93c60e50 100644 --- a/testing/constraints-3.14.txt +++ b/testing/constraints-3.14.txt @@ -10,4 +10,3 @@ google-auth>=2 grpcio>=1 proto-plus>=1 protobuf>=6 -grpc-google-iam-v1>=0 \ No newline at end of file diff --git a/testing/constraints-3.8.txt b/testing/constraints-3.8.txt index ef1c92ffff..93e6826f2a 100644 --- a/testing/constraints-3.8.txt +++ b/testing/constraints-3.8.txt @@ -6,4 +6,3 @@ google-auth grpcio proto-plus protobuf -grpc-google-iam-v1 diff --git a/testing/constraints-3.9.txt b/testing/constraints-3.9.txt index ef1c92ffff..93e6826f2a 100644 --- a/testing/constraints-3.9.txt +++ b/testing/constraints-3.9.txt @@ -6,4 +6,3 @@ google-auth grpcio proto-plus protobuf -grpc-google-iam-v1 diff --git a/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py b/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py index e210da1d37..ceade23bb0 100644 --- a/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py +++ b/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py @@ -184,12 +184,19 @@ def test__read_environment_variables(): with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): - with pytest.raises(ValueError) as excinfo: - DatabaseAdminClient._read_environment_variables() - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with pytest.raises(ValueError) as excinfo: + DatabaseAdminClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + else: + assert DatabaseAdminClient._read_environment_variables() == ( + False, + "auto", + None, + ) with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): assert DatabaseAdminClient._read_environment_variables() == ( @@ -228,6 +235,105 @@ def test__read_environment_variables(): ) +def test_use_client_cert_effective(): + # Test case 1: Test when `should_use_client_cert` returns True. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=True + ): + assert DatabaseAdminClient._use_client_cert_effective() is True + + # Test case 2: Test when `should_use_client_cert` returns False. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should NOT be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=False + ): + assert DatabaseAdminClient._use_client_cert_effective() is False + + # Test case 3: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "true". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert DatabaseAdminClient._use_client_cert_effective() is True + + # Test case 4: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert DatabaseAdminClient._use_client_cert_effective() is False + + # Test case 5: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "True". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "True"}): + assert DatabaseAdminClient._use_client_cert_effective() is True + + # Test case 6: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "False". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "False"} + ): + assert DatabaseAdminClient._use_client_cert_effective() is False + + # Test case 7: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "TRUE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "TRUE"}): + assert DatabaseAdminClient._use_client_cert_effective() is True + + # Test case 8: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "FALSE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "FALSE"} + ): + assert DatabaseAdminClient._use_client_cert_effective() is False + + # Test case 9: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not set. + # In this case, the method should return False, which is the default value. + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, clear=True): + assert DatabaseAdminClient._use_client_cert_effective() is False + + # Test case 10: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should raise a ValueError as the environment variable must be either + # "true" or "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + with pytest.raises(ValueError): + DatabaseAdminClient._use_client_cert_effective() + + # Test case 11: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should return False as the environment variable is set to an invalid value. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + assert DatabaseAdminClient._use_client_cert_effective() is False + + # Test case 12: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is unset. Also, + # the GOOGLE_API_CONFIG environment variable is unset. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": ""}): + with mock.patch.dict(os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": ""}): + assert DatabaseAdminClient._use_client_cert_effective() is False + + def test__get_client_cert_source(): mock_provided_cert_source = mock.Mock() mock_default_cert_source = mock.Mock() @@ -593,17 +699,6 @@ def test_database_admin_client_client_options( == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client = client_class(transport=transport_name) - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") with mock.patch.object(transport_class, "__init__") as patched: @@ -819,6 +914,119 @@ def test_database_admin_client_get_mtls_endpoint_and_cert_source(client_class): assert api_endpoint == mock_api_endpoint assert cert_source is None + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "Unsupported". + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source( + options + ) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset. + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", None) + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset(empty). + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() @@ -869,18 +1077,6 @@ def test_database_admin_client_get_mtls_endpoint_and_cert_source(client_class): == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client_class.get_mtls_endpoint_and_cert_source() - - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - @pytest.mark.parametrize( "client_class", [DatabaseAdminClient, DatabaseAdminAsyncClient] @@ -23552,6 +23748,7 @@ def test_database_admin_grpc_asyncio_transport_channel(): # Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are # removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.filterwarnings("ignore::FutureWarning") @pytest.mark.parametrize( "transport_class", [ diff --git a/tests/unit/gapic/spanner_admin_instance_v1/test_instance_admin.py b/tests/unit/gapic/spanner_admin_instance_v1/test_instance_admin.py index 532014af96..d8541c2be3 100644 --- a/tests/unit/gapic/spanner_admin_instance_v1/test_instance_admin.py +++ b/tests/unit/gapic/spanner_admin_instance_v1/test_instance_admin.py @@ -173,12 +173,19 @@ def test__read_environment_variables(): with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): - with pytest.raises(ValueError) as excinfo: - InstanceAdminClient._read_environment_variables() - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with pytest.raises(ValueError) as excinfo: + InstanceAdminClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + else: + assert InstanceAdminClient._read_environment_variables() == ( + False, + "auto", + None, + ) with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): assert InstanceAdminClient._read_environment_variables() == ( @@ -217,6 +224,105 @@ def test__read_environment_variables(): ) +def test_use_client_cert_effective(): + # Test case 1: Test when `should_use_client_cert` returns True. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=True + ): + assert InstanceAdminClient._use_client_cert_effective() is True + + # Test case 2: Test when `should_use_client_cert` returns False. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should NOT be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=False + ): + assert InstanceAdminClient._use_client_cert_effective() is False + + # Test case 3: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "true". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert InstanceAdminClient._use_client_cert_effective() is True + + # Test case 4: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert InstanceAdminClient._use_client_cert_effective() is False + + # Test case 5: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "True". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "True"}): + assert InstanceAdminClient._use_client_cert_effective() is True + + # Test case 6: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "False". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "False"} + ): + assert InstanceAdminClient._use_client_cert_effective() is False + + # Test case 7: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "TRUE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "TRUE"}): + assert InstanceAdminClient._use_client_cert_effective() is True + + # Test case 8: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "FALSE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "FALSE"} + ): + assert InstanceAdminClient._use_client_cert_effective() is False + + # Test case 9: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not set. + # In this case, the method should return False, which is the default value. + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, clear=True): + assert InstanceAdminClient._use_client_cert_effective() is False + + # Test case 10: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should raise a ValueError as the environment variable must be either + # "true" or "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + with pytest.raises(ValueError): + InstanceAdminClient._use_client_cert_effective() + + # Test case 11: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should return False as the environment variable is set to an invalid value. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + assert InstanceAdminClient._use_client_cert_effective() is False + + # Test case 12: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is unset. Also, + # the GOOGLE_API_CONFIG environment variable is unset. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": ""}): + with mock.patch.dict(os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": ""}): + assert InstanceAdminClient._use_client_cert_effective() is False + + def test__get_client_cert_source(): mock_provided_cert_source = mock.Mock() mock_default_cert_source = mock.Mock() @@ -582,17 +688,6 @@ def test_instance_admin_client_client_options( == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client = client_class(transport=transport_name) - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") with mock.patch.object(transport_class, "__init__") as patched: @@ -808,6 +903,119 @@ def test_instance_admin_client_get_mtls_endpoint_and_cert_source(client_class): assert api_endpoint == mock_api_endpoint assert cert_source is None + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "Unsupported". + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source( + options + ) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset. + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", None) + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset(empty). + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() @@ -858,18 +1066,6 @@ def test_instance_admin_client_get_mtls_endpoint_and_cert_source(client_class): == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client_class.get_mtls_endpoint_and_cert_source() - - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - @pytest.mark.parametrize( "client_class", [InstanceAdminClient, InstanceAdminAsyncClient] @@ -18840,6 +19036,7 @@ def test_instance_admin_grpc_asyncio_transport_channel(): # Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are # removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.filterwarnings("ignore::FutureWarning") @pytest.mark.parametrize( "transport_class", [ diff --git a/tests/unit/gapic/spanner_v1/test_spanner.py b/tests/unit/gapic/spanner_v1/test_spanner.py index d71d85a443..3725489794 100644 --- a/tests/unit/gapic/spanner_v1/test_spanner.py +++ b/tests/unit/gapic/spanner_v1/test_spanner.py @@ -156,12 +156,19 @@ def test__read_environment_variables(): with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): - with pytest.raises(ValueError) as excinfo: - SpannerClient._read_environment_variables() - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with pytest.raises(ValueError) as excinfo: + SpannerClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + else: + assert SpannerClient._read_environment_variables() == ( + False, + "auto", + None, + ) with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): assert SpannerClient._read_environment_variables() == (False, "never", None) @@ -184,6 +191,105 @@ def test__read_environment_variables(): assert SpannerClient._read_environment_variables() == (False, "auto", "foo.com") +def test_use_client_cert_effective(): + # Test case 1: Test when `should_use_client_cert` returns True. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=True + ): + assert SpannerClient._use_client_cert_effective() is True + + # Test case 2: Test when `should_use_client_cert` returns False. + # We mock the `should_use_client_cert` function to simulate a scenario where + # the google-auth library supports automatic mTLS and determines that a + # client certificate should NOT be used. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch( + "google.auth.transport.mtls.should_use_client_cert", return_value=False + ): + assert SpannerClient._use_client_cert_effective() is False + + # Test case 3: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "true". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert SpannerClient._use_client_cert_effective() is True + + # Test case 4: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"} + ): + assert SpannerClient._use_client_cert_effective() is False + + # Test case 5: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "True". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "True"}): + assert SpannerClient._use_client_cert_effective() is True + + # Test case 6: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "False". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "False"} + ): + assert SpannerClient._use_client_cert_effective() is False + + # Test case 7: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "TRUE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "TRUE"}): + assert SpannerClient._use_client_cert_effective() is True + + # Test case 8: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "FALSE". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "FALSE"} + ): + assert SpannerClient._use_client_cert_effective() is False + + # Test case 9: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not set. + # In this case, the method should return False, which is the default value. + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, clear=True): + assert SpannerClient._use_client_cert_effective() is False + + # Test case 10: Test when `should_use_client_cert` is unavailable and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should raise a ValueError as the environment variable must be either + # "true" or "false". + if not hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + with pytest.raises(ValueError): + SpannerClient._use_client_cert_effective() + + # Test case 11: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to an invalid value. + # The method should return False as the environment variable is set to an invalid value. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "unsupported"} + ): + assert SpannerClient._use_client_cert_effective() is False + + # Test case 12: Test when `should_use_client_cert` is available and the + # `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is unset. Also, + # the GOOGLE_API_CONFIG environment variable is unset. + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": ""}): + with mock.patch.dict(os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": ""}): + assert SpannerClient._use_client_cert_effective() is False + + def test__get_client_cert_source(): mock_provided_cert_source = mock.Mock() mock_default_cert_source = mock.Mock() @@ -539,17 +645,6 @@ def test_spanner_client_client_options(client_class, transport_class, transport_ == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client = client_class(transport=transport_name) - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") with mock.patch.object(transport_class, "__init__") as patched: @@ -764,6 +859,119 @@ def test_spanner_client_get_mtls_endpoint_and_cert_source(client_class): assert api_endpoint == mock_api_endpoint assert cert_source is None + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "Unsupported". + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source( + options + ) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset. + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", None) + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + + # Test cases for mTLS enablement when GOOGLE_API_USE_CLIENT_CERTIFICATE is unset(empty). + test_cases = [ + ( + # With workloads present in config, mTLS is enabled. + { + "version": 1, + "cert_configs": { + "workload": { + "cert_path": "path/to/cert/file", + "key_path": "path/to/key/file", + } + }, + }, + mock_client_cert_source, + ), + ( + # With workloads not present in config, mTLS is disabled. + { + "version": 1, + "cert_configs": {}, + }, + None, + ), + ] + if hasattr(google.auth.transport.mtls, "should_use_client_cert"): + for config_data, expected_cert_source in test_cases: + env = os.environ.copy() + env.pop("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") + with mock.patch.dict(os.environ, env, clear=True): + config_filename = "mock_certificate_config.json" + config_file_content = json.dumps(config_data) + m = mock.mock_open(read_data=config_file_content) + with mock.patch("builtins.open", m): + with mock.patch.dict( + os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} + ): + mock_api_endpoint = "foo" + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, + api_endpoint=mock_api_endpoint, + ) + ( + api_endpoint, + cert_source, + ) = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is expected_cert_source + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() @@ -814,18 +1022,6 @@ def test_spanner_client_get_mtls_endpoint_and_cert_source(client_class): == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" ) - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} - ): - with pytest.raises(ValueError) as excinfo: - client_class.get_mtls_endpoint_and_cert_source() - - assert ( - str(excinfo.value) - == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" - ) - @pytest.mark.parametrize("client_class", [SpannerClient, SpannerAsyncClient]) @mock.patch.object( @@ -12382,6 +12578,7 @@ def test_spanner_grpc_asyncio_transport_channel(): # Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are # removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.filterwarnings("ignore::FutureWarning") @pytest.mark.parametrize( "transport_class", [transports.SpannerGrpcTransport, transports.SpannerGrpcAsyncIOTransport], From 2acea5e2d39b2fcfb5a07f2bbd1383c9b08f940a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 16 Dec 2025 20:50:52 -0800 Subject: [PATCH 363/364] chore: librarian release pull request: 20251216T134400Z (#1468) PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. Librarian Version: v0.7.0 Language Image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:b8058df4c45e9a6e07f6b4d65b458d0d059241dd34c814f151c8bf6b89211209
google-cloud-spanner: 3.61.0 ## [3.61.0](https://github.com/googleapis/python-spanner/compare/v3.60.0...v3.61.0) (2025-12-16) ### Features * support mTLS certificates when available (#1467) ([df87c3ed](https://github.com/googleapis/python-spanner/commit/df87c3ed))
--- .librarian/state.yaml | 2 +- CHANGELOG.md | 7 +++++++ google/cloud/spanner_admin_database_v1/gapic_version.py | 2 +- google/cloud/spanner_admin_instance_v1/gapic_version.py | 2 +- google/cloud/spanner_dbapi/version.py | 2 +- google/cloud/spanner_v1/gapic_version.py | 2 +- .../snippet_metadata_google.spanner.admin.database.v1.json | 2 +- .../snippet_metadata_google.spanner.admin.instance.v1.json | 2 +- .../snippet_metadata_google.spanner.v1.json | 2 +- 9 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.librarian/state.yaml b/.librarian/state.yaml index 6fb8be06a7..381824b372 100644 --- a/.librarian/state.yaml +++ b/.librarian/state.yaml @@ -1,7 +1,7 @@ image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:b8058df4c45e9a6e07f6b4d65b458d0d059241dd34c814f151c8bf6b89211209 libraries: - id: google-cloud-spanner - version: 3.60.0 + version: 3.61.0 last_generated_commit: a17b84add8318f780fcc8a027815d5fee644b9f7 apis: - path: google/spanner/admin/instance/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a5a487e85..73b4a8d8d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-cloud-spanner/#history +## [3.61.0](https://github.com/googleapis/python-spanner/compare/v3.60.0...v3.61.0) (2025-12-16) + + +### Features + +* support mTLS certificates when available (#1467) ([df87c3ed55db7cffa2eed4d7316ca5c375af1c5a](https://github.com/googleapis/python-spanner/commit/df87c3ed55db7cffa2eed4d7316ca5c375af1c5a)) + ## [3.60.0](https://github.com/googleapis/python-spanner/compare/v3.59.0...v3.60.0) (2025-12-10) diff --git a/google/cloud/spanner_admin_database_v1/gapic_version.py b/google/cloud/spanner_admin_database_v1/gapic_version.py index 992322a033..89cb359ff2 100644 --- a/google/cloud/spanner_admin_database_v1/gapic_version.py +++ b/google/cloud/spanner_admin_database_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "3.60.0" # {x-release-please-version} +__version__ = "3.61.0" # {x-release-please-version} diff --git a/google/cloud/spanner_admin_instance_v1/gapic_version.py b/google/cloud/spanner_admin_instance_v1/gapic_version.py index 992322a033..89cb359ff2 100644 --- a/google/cloud/spanner_admin_instance_v1/gapic_version.py +++ b/google/cloud/spanner_admin_instance_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "3.60.0" # {x-release-please-version} +__version__ = "3.61.0" # {x-release-please-version} diff --git a/google/cloud/spanner_dbapi/version.py b/google/cloud/spanner_dbapi/version.py index ee7431572b..86252a8635 100644 --- a/google/cloud/spanner_dbapi/version.py +++ b/google/cloud/spanner_dbapi/version.py @@ -15,6 +15,6 @@ import platform PY_VERSION = platform.python_version() -__version__ = "3.60.0" +__version__ = "3.61.0" VERSION = __version__ DEFAULT_USER_AGENT = "gl-dbapi/" + VERSION diff --git a/google/cloud/spanner_v1/gapic_version.py b/google/cloud/spanner_v1/gapic_version.py index 992322a033..89cb359ff2 100644 --- a/google/cloud/spanner_v1/gapic_version.py +++ b/google/cloud/spanner_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "3.60.0" # {x-release-please-version} +__version__ = "3.61.0" # {x-release-please-version} diff --git a/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json b/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json index 0bfe97d988..4fd6fa5396 100644 --- a/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json +++ b/samples/generated_samples/snippet_metadata_google.spanner.admin.database.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-spanner-admin-database", - "version": "3.60.0" + "version": "3.61.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json b/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json index 9b51de3471..bae057d766 100644 --- a/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json +++ b/samples/generated_samples/snippet_metadata_google.spanner.admin.instance.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-spanner-admin-instance", - "version": "3.60.0" + "version": "3.61.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.spanner.v1.json b/samples/generated_samples/snippet_metadata_google.spanner.v1.json index 1ec5a82e5a..5148cfa6df 100644 --- a/samples/generated_samples/snippet_metadata_google.spanner.v1.json +++ b/samples/generated_samples/snippet_metadata_google.spanner.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-spanner", - "version": "3.60.0" + "version": "3.61.0" }, "snippets": [ { From 3d3cea0b5afb414a506ab08eebae733d803f17ac Mon Sep 17 00:00:00 2001 From: surbhigarg92 Date: Wed, 17 Dec 2025 11:11:09 +0530 Subject: [PATCH 364/364] fix: transaction_tag should be set on BeginTransactionRequest (#1463) When using multiplexed sessions, the transaction_tag should also be set on the BeginTransactionRequest. --------- Co-authored-by: rahul2393 --- google/cloud/spanner_v1/snapshot.py | 23 +++++++++++++++++++---- google/cloud/spanner_v1/transaction.py | 4 +++- tests/unit/test_session.py | 5 ++++- tests/unit/test_transaction.py | 1 + 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/google/cloud/spanner_v1/snapshot.py b/google/cloud/spanner_v1/snapshot.py index 46b0f5af8d..89cbc9fe88 100644 --- a/google/cloud/spanner_v1/snapshot.py +++ b/google/cloud/spanner_v1/snapshot.py @@ -901,13 +901,19 @@ def attempt_tracking_method(): return [partition.partition_token for partition in response.partitions] - def _begin_transaction(self, mutation: Mutation = None) -> bytes: + def _begin_transaction( + self, mutation: Mutation = None, transaction_tag: str = None + ) -> bytes: """Begins a transaction on the database. :type mutation: :class:`~google.cloud.spanner_v1.mutation.Mutation` :param mutation: (Optional) Mutation to include in the begin transaction request. Required for mutation-only transactions with multiplexed sessions. + :type transaction_tag: str + :param transaction_tag: (Optional) Transaction tag to include in the begin transaction + request. + :rtype: bytes :returns: identifier for the transaction. @@ -931,6 +937,17 @@ def _begin_transaction(self, mutation: Mutation = None) -> bytes: (_metadata_with_leader_aware_routing(database._route_to_leader_enabled)) ) + begin_request_kwargs = { + "session": session.name, + "options": self._build_transaction_selector_pb().begin, + "mutation_key": mutation, + } + + if transaction_tag: + begin_request_kwargs["request_options"] = RequestOptions( + transaction_tag=transaction_tag + ) + with trace_call( name=f"CloudSpanner.{type(self).__name__}.begin", session=session, @@ -942,9 +959,7 @@ def _begin_transaction(self, mutation: Mutation = None) -> bytes: def wrapped_method(): begin_transaction_request = BeginTransactionRequest( - session=session.name, - options=self._build_transaction_selector_pb().begin, - mutation_key=mutation, + **begin_request_kwargs ) begin_transaction_method = functools.partial( api.begin_transaction, diff --git a/google/cloud/spanner_v1/transaction.py b/google/cloud/spanner_v1/transaction.py index b9e14a0040..de8b421840 100644 --- a/google/cloud/spanner_v1/transaction.py +++ b/google/cloud/spanner_v1/transaction.py @@ -714,7 +714,9 @@ def _begin_transaction(self, mutation: Mutation = None) -> bytes: if self.rolled_back: raise ValueError("Transaction is already rolled back") - return super(Transaction, self)._begin_transaction(mutation=mutation) + return super(Transaction, self)._begin_transaction( + mutation=mutation, transaction_tag=self.transaction_tag + ) def _begin_mutations_only_transaction(self) -> None: """Begins a mutations-only transaction on the database.""" diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index bfbd6edd5e..8026c50c24 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -2005,9 +2005,12 @@ def unit_of_work(txn, *args, **kw): self.assertEqual(kw, {"some_arg": "def"}) expected_options = TransactionOptions(read_write=TransactionOptions.ReadWrite()) + expected_request_options = RequestOptions(transaction_tag=transaction_tag) gax_api.begin_transaction.assert_called_once_with( request=BeginTransactionRequest( - session=self.SESSION_NAME, options=expected_options + session=self.SESSION_NAME, + options=expected_options, + request_options=expected_request_options, ), metadata=[ ("google-cloud-resource-prefix", database.name), diff --git a/tests/unit/test_transaction.py b/tests/unit/test_transaction.py index 39656cb8d1..510251656e 100644 --- a/tests/unit/test_transaction.py +++ b/tests/unit/test_transaction.py @@ -467,6 +467,7 @@ def _commit_helper( session=session.name, options=TransactionOptions(read_write=TransactionOptions.ReadWrite()), mutation_key=expected_begin_mutation, + request_options=RequestOptions(transaction_tag=TRANSACTION_TAG), ) expected_begin_metadata = base_metadata.copy()