diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 50b29ff..58fcbee 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:b2dc5f80edcf5d4486c39068c9fa11f7f851d9568eea4dcba130f994ea9b5e97 -# created: 2022-06-12T16:09:31.61859086Z + digest: sha256:c8878270182edaab99f2927969d4f700c3af265accd472c3425deedff2b7fd93 +# created: 2022-07-14T01:58:16.015625351Z diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index e5be6ed..5531b01 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python: ['3.7', '3.8', '3.9', '3.10'] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg deleted file mode 100644 index e13bc43..0000000 --- a/.kokoro/samples/python3.6/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.6" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py36" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-migration/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-migration/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/continuous.cfg b/.kokoro/samples/python3.6/continuous.cfg deleted file mode 100644 index 7218af1..0000000 --- a/.kokoro/samples/python3.6/continuous.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg deleted file mode 100644 index 3b9b646..0000000 --- a/.kokoro/samples/python3.6/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-migration/.kokoro/test-samples-against-head.sh" -} diff --git a/.kokoro/samples/python3.6/periodic.cfg b/.kokoro/samples/python3.6/periodic.cfg deleted file mode 100644 index 71cd1e5..0000000 --- a/.kokoro/samples/python3.6/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/.kokoro/samples/python3.6/presubmit.cfg b/.kokoro/samples/python3.6/presubmit.cfg deleted file mode 100644 index a1c8d97..0000000 --- a/.kokoro/samples/python3.6/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 8a324c9..2c6500c 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -33,7 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.6 -m pip install --upgrade --quiet nox +python3.9 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then @@ -76,7 +76,7 @@ for file in samples/**/requirements.txt; do echo "------------------------------------------------------------" # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" + python3.9 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cfe463..59d5eb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [0.6.0](https://github.com/googleapis/python-bigquery-migration/compare/v0.5.0...v0.6.0) (2022-07-16) + + +### Features + +* add audience parameter ([77e4c52](https://github.com/googleapis/python-bigquery-migration/commit/77e4c52d597e62956e2349dfff2f1b88df013450)) +* Add Presto and Postgresql dialects ([#103](https://github.com/googleapis/python-bigquery-migration/issues/103)) ([77e4c52](https://github.com/googleapis/python-bigquery-migration/commit/77e4c52d597e62956e2349dfff2f1b88df013450)) + + +### Bug Fixes + +* **deps:** require google-api-core>=1.32.0,>=2.8.0 ([77e4c52](https://github.com/googleapis/python-bigquery-migration/commit/77e4c52d597e62956e2349dfff2f1b88df013450)) +* require python 3.7+ ([#105](https://github.com/googleapis/python-bigquery-migration/issues/105)) ([13f50a7](https://github.com/googleapis/python-bigquery-migration/commit/13f50a7b845f5d4ce0aa03821578b730ea37dbb2)) + ## [0.5.0](https://github.com/googleapis/python-bigquery-migration/compare/v0.4.3...v0.5.0) (2022-06-15) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d02d709..2632ba4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.6, 3.7, 3.8, 3.9 and 3.10 on both UNIX and Windows. + 3.7, 3.8, 3.9 and 3.10 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -221,13 +221,11 @@ Supported Python Versions We support: -- `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ - `Python 3.10`_ -.. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ @@ -239,7 +237,7 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-bigquery-migration/blob/main/noxfile.py -We also explicitly decided to support Python 3 beginning with version 3.6. +We also explicitly decided to support Python 3 beginning with version 3.7. Reasons for this include: - Encouraging use of newest versions of Python 3 diff --git a/README.rst b/README.rst index 16ea38a..63cfd7e 100644 --- a/README.rst +++ b/README.rst @@ -1,20 +1,20 @@ -Python Client for Google Cloud Bigquery Migration API -===================================================== +Python Client for Google BigQuery Migration API +=============================================== -|alpha| |pypi| |versions| +|preview| |pypi| |versions| -Python client for `Google Cloud Bigquery Migration API`_. +`Google BigQuery Migration API`_: - `Client Library Documentation`_ - `Product Documentation`_ -.. |alpha| image:: https://img.shields.io/badge/support-alpha-orange.svg - :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability +.. |preview| image:: https://img.shields.io/badge/support-preview-orange.svg + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#stability-levels .. |pypi| image:: https://img.shields.io/pypi/v/google-cloud-bigquery-migration.svg :target: https://pypi.org/project/google-cloud-bigquery-migration/ .. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-bigquery-migration.svg :target: https://pypi.org/project/google-cloud-bigquery-migration/ -.. _Google Cloud Bigquery Migration API: https://cloud.google.com/bigquery/docs/reference/migration/ +.. _Google BigQuery Migration API: https://cloud.google.com/bigquery/docs/reference/migration/ .. _Client Library Documentation: https://cloud.google.com/python/docs/reference/bigquerymigration/latest .. _Product Documentation: https://cloud.google.com/bigquery/docs/reference/migration/ @@ -25,12 +25,12 @@ In order to use this library, you first need to go through the following steps: 1. `Select or create a Cloud Platform project.`_ 2. `Enable billing for your project.`_ -3. `Enable the Google Cloud Bigquery Migration API.`_ +3. `Enable the Google BigQuery Migration API.`_ 4. `Setup Authentication.`_ .. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project .. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project -.. _Enable the Google Cloud Bigquery Migration API.: https://console.cloud.google.com/apis/library/bigquerymigration.googleapis.com +.. _Enable the Google BigQuery Migration API.: https://cloud.google.com/bigquery/docs/reference/migration/ .. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html Installation @@ -47,6 +47,31 @@ dependencies. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ +Code samples and snippets +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Code samples and snippets live in the `samples/` folder. + + +Supported Python Versions +^^^^^^^^^^^^^^^^^^^^^^^^^ +Our client libraries are compatible with all current `active`_ and `maintenance`_ versions of +Python. + +Python >= 3.7 + +.. _active: https://devguide.python.org/devcycle/#in-development-main-branch +.. _maintenance: https://devguide.python.org/devcycle/#maintenance-branches + +Unsupported Python Versions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Python <= 3.6 + +If you are using an `end-of-life`_ +version of Python, we recommend that you update as soon as possible to an actively supported version. + +.. _end-of-life: https://devguide.python.org/devcycle/#end-of-life-branches + Mac/Linux ^^^^^^^^^ @@ -71,12 +96,12 @@ Windows Next Steps ~~~~~~~~~~ -- Read the `Client Library Documentation`_ for Google Cloud Bigquery - Migration API to see other available methods on the client. -- Read the `Bigquery Migration API Product documentation`_ to learn +- Read the `Client Library Documentation`_ for Google BigQuery Migration API + to see other available methods on the client. +- Read the `Google BigQuery Migration API Product documentation`_ to learn more about the product and see How-to Guides. - View this `README`_ to see the full list of Cloud APIs that we cover. -.. _Bigquery Migration API Product documentation: https://cloud.google.com/bigquery/docs/reference/migration/ +.. _Google BigQuery Migration API Product documentation: https://cloud.google.com/bigquery/docs/reference/migration/ .. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst diff --git a/google/cloud/bigquery_migration/__init__.py b/google/cloud/bigquery_migration/__init__.py index f9602f0..7c12438 100644 --- a/google/cloud/bigquery_migration/__init__.py +++ b/google/cloud/bigquery_migration/__init__.py @@ -58,6 +58,8 @@ ObjectNameMapping, ObjectNameMappingList, OracleDialect, + PostgresqlDialect, + PrestoDialect, RedshiftDialect, SnowflakeDialect, SourceEnv, @@ -100,6 +102,8 @@ "ObjectNameMapping", "ObjectNameMappingList", "OracleDialect", + "PostgresqlDialect", + "PrestoDialect", "RedshiftDialect", "SnowflakeDialect", "SourceEnv", diff --git a/google/cloud/bigquery_migration_v2/__init__.py b/google/cloud/bigquery_migration_v2/__init__.py index 0ec86fe..76c30df 100644 --- a/google/cloud/bigquery_migration_v2/__init__.py +++ b/google/cloud/bigquery_migration_v2/__init__.py @@ -47,6 +47,8 @@ ObjectNameMapping, ObjectNameMappingList, OracleDialect, + PostgresqlDialect, + PrestoDialect, RedshiftDialect, SnowflakeDialect, SourceEnv, @@ -84,6 +86,8 @@ "ObjectNameMappingList", "OracleDialect", "Point", + "PostgresqlDialect", + "PrestoDialect", "RedshiftDialect", "ResourceErrorDetail", "SQLServerDialect", diff --git a/google/cloud/bigquery_migration_v2/services/migration_service/client.py b/google/cloud/bigquery_migration_v2/services/migration_service/client.py index de6faad..576bd42 100644 --- a/google/cloud/bigquery_migration_v2/services/migration_service/client.py +++ b/google/cloud/bigquery_migration_v2/services/migration_service/client.py @@ -457,6 +457,7 @@ def __init__( quota_project_id=client_options.quota_project_id, client_info=client_info, always_use_jwt_access=True, + api_audience=client_options.api_audience, ) def create_migration_workflow( diff --git a/google/cloud/bigquery_migration_v2/services/migration_service/transports/base.py b/google/cloud/bigquery_migration_v2/services/migration_service/transports/base.py index fe28c31..5c92cf1 100644 --- a/google/cloud/bigquery_migration_v2/services/migration_service/transports/base.py +++ b/google/cloud/bigquery_migration_v2/services/migration_service/transports/base.py @@ -58,6 +58,7 @@ def __init__( quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, **kwargs, ) -> None: """Instantiate the transport. @@ -85,11 +86,6 @@ def __init__( be used for service account credentials. """ - # Save the hostname. Default to port 443 (HTTPS) if none is specified. - if ":" not in host: - host += ":443" - self._host = host - scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} # Save the scopes. @@ -110,6 +106,11 @@ def __init__( credentials, _ = google.auth.default( **scopes_kwargs, quota_project_id=quota_project_id ) + # Don't apply audience if the credentials file passed from user. + if hasattr(credentials, "with_gdch_audience"): + credentials = credentials.with_gdch_audience( + api_audience if api_audience else host + ) # If the credentials are service account credentials, then always try to use self signed JWT. if ( @@ -122,6 +123,11 @@ def __init__( # Save the credentials. self._credentials = credentials + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ":" not in host: + host += ":443" + self._host = host + def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { diff --git a/google/cloud/bigquery_migration_v2/services/migration_service/transports/grpc.py b/google/cloud/bigquery_migration_v2/services/migration_service/transports/grpc.py index 93a4193..930489f 100644 --- a/google/cloud/bigquery_migration_v2/services/migration_service/transports/grpc.py +++ b/google/cloud/bigquery_migration_v2/services/migration_service/transports/grpc.py @@ -61,6 +61,7 @@ def __init__( quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, ) -> None: """Instantiate the transport. @@ -156,6 +157,7 @@ def __init__( quota_project_id=quota_project_id, client_info=client_info, always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, ) if not self._grpc_channel: diff --git a/google/cloud/bigquery_migration_v2/services/migration_service/transports/grpc_asyncio.py b/google/cloud/bigquery_migration_v2/services/migration_service/transports/grpc_asyncio.py index bc6d578..db6e440 100644 --- a/google/cloud/bigquery_migration_v2/services/migration_service/transports/grpc_asyncio.py +++ b/google/cloud/bigquery_migration_v2/services/migration_service/transports/grpc_asyncio.py @@ -106,6 +106,7 @@ def __init__( quota_project_id=None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, ) -> None: """Instantiate the transport. @@ -201,6 +202,7 @@ def __init__( quota_project_id=quota_project_id, client_info=client_info, always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, ) if not self._grpc_channel: diff --git a/google/cloud/bigquery_migration_v2/types/__init__.py b/google/cloud/bigquery_migration_v2/types/__init__.py index 74a7d77..45476e5 100644 --- a/google/cloud/bigquery_migration_v2/types/__init__.py +++ b/google/cloud/bigquery_migration_v2/types/__init__.py @@ -38,6 +38,8 @@ ObjectNameMapping, ObjectNameMappingList, OracleDialect, + PostgresqlDialect, + PrestoDialect, RedshiftDialect, SnowflakeDialect, SourceEnv, @@ -78,6 +80,8 @@ "ObjectNameMapping", "ObjectNameMappingList", "OracleDialect", + "PostgresqlDialect", + "PrestoDialect", "RedshiftDialect", "SnowflakeDialect", "SourceEnv", diff --git a/google/cloud/bigquery_migration_v2/types/migration_entities.py b/google/cloud/bigquery_migration_v2/types/migration_entities.py index 883c985..c0b3c18 100644 --- a/google/cloud/bigquery_migration_v2/types/migration_entities.py +++ b/google/cloud/bigquery_migration_v2/types/migration_entities.py @@ -122,7 +122,7 @@ class MigrationTask(proto.Message): Translation_HiveQL2BQ, Translation_SparkSQL2BQ, Translation_Snowflake2BQ, Translation_Netezza2BQ, Translation_AzureSynapse2BQ, Translation_Vertica2BQ, - Translation_SQLServer2BQ. + Translation_SQLServer2BQ, Translation_Presto2BQ. state (google.cloud.bigquery_migration_v2.types.MigrationTask.State): Output only. The current state of the task. processing_error (google.rpc.error_details_pb2.ErrorInfo): diff --git a/google/cloud/bigquery_migration_v2/types/translation_config.py b/google/cloud/bigquery_migration_v2/types/translation_config.py index 4545a58..bab0fe8 100644 --- a/google/cloud/bigquery_migration_v2/types/translation_config.py +++ b/google/cloud/bigquery_migration_v2/types/translation_config.py @@ -31,6 +31,8 @@ "AzureSynapseDialect", "VerticaDialect", "SQLServerDialect", + "PostgresqlDialect", + "PrestoDialect", "ObjectNameMappingList", "ObjectNameMapping", "NameMappingKey", @@ -160,6 +162,14 @@ class Dialect(proto.Message): sql_server_dialect (google.cloud.bigquery_migration_v2.types.SQLServerDialect): The SQL Server dialect + This field is a member of `oneof`_ ``dialect_value``. + postgresql_dialect (google.cloud.bigquery_migration_v2.types.PostgresqlDialect): + The Postgresql dialect + + This field is a member of `oneof`_ ``dialect_value``. + presto_dialect (google.cloud.bigquery_migration_v2.types.PrestoDialect): + The Presto dialect + This field is a member of `oneof`_ ``dialect_value``. """ @@ -229,6 +239,18 @@ class Dialect(proto.Message): oneof="dialect_value", message="SQLServerDialect", ) + postgresql_dialect = proto.Field( + proto.MESSAGE, + number=12, + oneof="dialect_value", + message="PostgresqlDialect", + ) + presto_dialect = proto.Field( + proto.MESSAGE, + number=13, + oneof="dialect_value", + message="PrestoDialect", + ) class BigQueryDialect(proto.Message): @@ -293,6 +315,14 @@ class SQLServerDialect(proto.Message): r"""The dialect definition for SQL Server.""" +class PostgresqlDialect(proto.Message): + r"""The dialect definition for Postgresql.""" + + +class PrestoDialect(proto.Message): + r"""The dialect definition for Presto.""" + + class ObjectNameMappingList(proto.Message): r"""Represents a map of name mappings using a list of key:value proto messages of existing name to desired output name. diff --git a/google/cloud/bigquery_migration_v2alpha/services/migration_service/client.py b/google/cloud/bigquery_migration_v2alpha/services/migration_service/client.py index 1c545c3..d4e669a 100644 --- a/google/cloud/bigquery_migration_v2alpha/services/migration_service/client.py +++ b/google/cloud/bigquery_migration_v2alpha/services/migration_service/client.py @@ -457,6 +457,7 @@ def __init__( quota_project_id=client_options.quota_project_id, client_info=client_info, always_use_jwt_access=True, + api_audience=client_options.api_audience, ) def create_migration_workflow( diff --git a/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/base.py b/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/base.py index 083f473..e59d9ea 100644 --- a/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/base.py +++ b/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/base.py @@ -58,6 +58,7 @@ def __init__( quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, **kwargs, ) -> None: """Instantiate the transport. @@ -85,11 +86,6 @@ def __init__( be used for service account credentials. """ - # Save the hostname. Default to port 443 (HTTPS) if none is specified. - if ":" not in host: - host += ":443" - self._host = host - scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} # Save the scopes. @@ -110,6 +106,11 @@ def __init__( credentials, _ = google.auth.default( **scopes_kwargs, quota_project_id=quota_project_id ) + # Don't apply audience if the credentials file passed from user. + if hasattr(credentials, "with_gdch_audience"): + credentials = credentials.with_gdch_audience( + api_audience if api_audience else host + ) # If the credentials are service account credentials, then always try to use self signed JWT. if ( @@ -122,6 +123,11 @@ def __init__( # Save the credentials. self._credentials = credentials + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ":" not in host: + host += ":443" + self._host = host + def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { diff --git a/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/grpc.py b/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/grpc.py index a295cda..1f4729b 100644 --- a/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/grpc.py +++ b/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/grpc.py @@ -61,6 +61,7 @@ def __init__( quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, ) -> None: """Instantiate the transport. @@ -156,6 +157,7 @@ def __init__( quota_project_id=quota_project_id, client_info=client_info, always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, ) if not self._grpc_channel: diff --git a/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/grpc_asyncio.py b/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/grpc_asyncio.py index 3b2b936..3bf5e0c 100644 --- a/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/grpc_asyncio.py +++ b/google/cloud/bigquery_migration_v2alpha/services/migration_service/transports/grpc_asyncio.py @@ -106,6 +106,7 @@ def __init__( quota_project_id=None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, ) -> None: """Instantiate the transport. @@ -201,6 +202,7 @@ def __init__( quota_project_id=quota_project_id, client_info=client_info, always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, ) if not self._grpc_channel: diff --git a/noxfile.py b/noxfile.py index 5bb3c20..94b2f9c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -32,7 +32,7 @@ DEFAULT_PYTHON_VERSION = "3.8" -UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", @@ -332,28 +332,15 @@ def docfx(session): def prerelease_deps(session): """Run all tests with prerelease versions of dependencies installed.""" - prerel_deps = [ - "protobuf", - "googleapis-common-protos", - "google-auth", - "grpcio", - "grpcio-status", - "google-api-core", - "proto-plus", - # dependencies of google-auth - "cryptography", - "pyasn1", - ] - - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = ["requests"] - session.install(*other_deps) - + # Install all dependencies + session.install("-e", ".[all, tests, tracing]") session.install(*UNIT_TEST_STANDARD_DEPENDENCIES) - session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) + system_deps_all = ( + SYSTEM_TEST_STANDARD_DEPENDENCIES + + SYSTEM_TEST_EXTERNAL_DEPENDENCIES + + SYSTEM_TEST_EXTRAS + ) + session.install(*system_deps_all) # Because we test minimum dependency versions on the minimum Python # version, the first version we test with in the unit tests sessions has a @@ -367,19 +354,44 @@ def prerelease_deps(session): constraints_text = constraints_file.read() # Ignore leading whitespace and comment lines. - deps = [ + constraints_deps = [ match.group(1) for match in re.finditer( r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE ) ] - # Don't overwrite prerelease packages. - deps = [dep for dep in deps if dep not in prerel_deps] - # We use --no-deps to ensure that pre-release versions aren't overwritten - # by the version ranges in setup.py. - session.install(*deps) - session.install("--no-deps", "-e", ".[all]") + session.install(*constraints_deps) + + if os.path.exists("samples/snippets/requirements.txt"): + session.install("-r", "samples/snippets/requirements.txt") + + if os.path.exists("samples/snippets/requirements-test.txt"): + session.install("-r", "samples/snippets/requirements-test.txt") + + prerel_deps = [ + "protobuf", + # dependency of grpc + "six", + "googleapis-common-protos", + "grpcio", + "grpcio-status", + "google-api-core", + "proto-plus", + "google-cloud-testutils", + # dependencies of google-cloud-testutils" + "click", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + "google-auth", + ] + session.install(*other_deps) # Print out prerelease package versions session.run( @@ -388,5 +400,16 @@ def prerelease_deps(session): session.run("python", "-c", "import grpc; print(grpc.__version__)") session.run("py.test", "tests/unit") - session.run("py.test", "tests/system") - session.run("py.test", "samples/snippets") + + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Only run system tests if found. + if os.path.exists(system_test_path) or os.path.exists(system_test_folder_path): + session.run("py.test", "tests/system") + + snippets_test_path = os.path.join("samples", "snippets") + + # Only run samples tests if found. + if os.path.exists(snippets_test_path): + session.run("py.test", "samples/snippets") diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 3b3ffa5..e9eb1cb 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -88,7 +88,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index 275d649..6f069c6 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 3.6+. +#. Create a virtualenv. Samples are compatible with Python 3.7+. .. code-block:: bash diff --git a/setup.py b/setup.py index 4cdaf63..998f1f6 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ import setuptools # type: ignore -version = "0.5.0" +version = "0.6.0" package_root = os.path.abspath(os.path.dirname(__file__)) @@ -39,17 +39,16 @@ platforms="Posix; MacOS X; Windows", include_package_data=True, install_requires=( - "google-api-core[grpc] >= 1.31.5, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0", + "google-api-core[grpc] >= 1.32.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*", "proto-plus >= 1.15.0, <2.0.0dev", "protobuf >= 3.19.0, <4.0.0dev", ), - python_requires=">=3.6", + python_requires=">=3.7", classifiers=[ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt deleted file mode 100644 index 3b53664..0000000 --- a/testing/constraints-3.6.txt +++ /dev/null @@ -1,11 +0,0 @@ -# This constraints file is used to check that lower bounds -# are correct in setup.py -# List *all* library dependencies and extras in this file. -# Pin the version to the lower bound. -# -# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", -# Then this file should have foo==1.14.0 -google-api-core==1.31.5 -proto-plus==1.15.0 -libcst==0.2.5 -protobuf==3.19.0 diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 3b53664..5d57a8d 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -5,7 +5,7 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -google-api-core==1.31.5 +google-api-core==1.32.0 proto-plus==1.15.0 libcst==0.2.5 protobuf==3.19.0 diff --git a/tests/unit/gapic/bigquery_migration_v2/test_migration_service.py b/tests/unit/gapic/bigquery_migration_v2/test_migration_service.py index 7037ae5..555129a 100644 --- a/tests/unit/gapic/bigquery_migration_v2/test_migration_service.py +++ b/tests/unit/gapic/bigquery_migration_v2/test_migration_service.py @@ -237,6 +237,7 @@ def test_migration_service_client_client_options( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is @@ -254,6 +255,7 @@ def test_migration_service_client_client_options( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is @@ -271,6 +273,7 @@ def test_migration_service_client_client_options( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has @@ -300,6 +303,25 @@ def test_migration_service_client_client_options( quota_project_id="octopus", client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, + ) + # Check the case api_endpoint is provided + options = client_options.ClientOptions( + api_audience="https://language.googleapis.com" + ) + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience="https://language.googleapis.com", ) @@ -377,6 +399,7 @@ def test_migration_service_client_mtls_env_auto( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case ADC client cert is provided. Whether client cert is used depends on @@ -411,6 +434,7 @@ def test_migration_service_client_mtls_env_auto( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case client_cert_source and ADC client cert are not provided. @@ -433,6 +457,7 @@ def test_migration_service_client_mtls_env_auto( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) @@ -547,6 +572,7 @@ def test_migration_service_client_client_options_scopes( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) @@ -585,6 +611,7 @@ def test_migration_service_client_client_options_credentials_file( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) @@ -605,6 +632,7 @@ def test_migration_service_client_client_options_from_dict(): quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) @@ -643,6 +671,7 @@ def test_migration_service_client_create_channel_credentials_file( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # test that the credentials from file are saved and used as the credentials. @@ -3042,6 +3071,28 @@ def test_migration_service_transport_auth_adc(transport_class): ) +@pytest.mark.parametrize( + "transport_class", + [ + transports.MigrationServiceGrpcTransport, + transports.MigrationServiceGrpcAsyncIOTransport, + ], +) +def test_migration_service_transport_auth_gdch_credentials(transport_class): + host = "https://language.com" + api_audience_tests = [None, "https://language2.com"] + api_audience_expect = [host, "https://language2.com"] + for t, e in zip(api_audience_tests, api_audience_expect): + with mock.patch.object(google.auth, "default", autospec=True) as adc: + gdch_mock = mock.MagicMock() + type(gdch_mock).with_gdch_audience = mock.PropertyMock( + return_value=gdch_mock + ) + adc.return_value = (gdch_mock, None) + transport_class(host=host, api_audience=t) + gdch_mock.with_gdch_audience.assert_called_once_with(e) + + @pytest.mark.parametrize( "transport_class,grpc_helpers", [ @@ -3541,4 +3592,5 @@ def test_api_key_credentials(client_class, transport_class): quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) diff --git a/tests/unit/gapic/bigquery_migration_v2alpha/test_migration_service.py b/tests/unit/gapic/bigquery_migration_v2alpha/test_migration_service.py index 3d14b15..bb31bcd 100644 --- a/tests/unit/gapic/bigquery_migration_v2alpha/test_migration_service.py +++ b/tests/unit/gapic/bigquery_migration_v2alpha/test_migration_service.py @@ -239,6 +239,7 @@ def test_migration_service_client_client_options( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is @@ -256,6 +257,7 @@ def test_migration_service_client_client_options( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is @@ -273,6 +275,7 @@ def test_migration_service_client_client_options( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has @@ -302,6 +305,25 @@ def test_migration_service_client_client_options( quota_project_id="octopus", client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, + ) + # Check the case api_endpoint is provided + options = client_options.ClientOptions( + api_audience="https://language.googleapis.com" + ) + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience="https://language.googleapis.com", ) @@ -379,6 +401,7 @@ def test_migration_service_client_mtls_env_auto( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case ADC client cert is provided. Whether client cert is used depends on @@ -413,6 +436,7 @@ def test_migration_service_client_mtls_env_auto( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # Check the case client_cert_source and ADC client cert are not provided. @@ -435,6 +459,7 @@ def test_migration_service_client_mtls_env_auto( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) @@ -549,6 +574,7 @@ def test_migration_service_client_client_options_scopes( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) @@ -587,6 +613,7 @@ def test_migration_service_client_client_options_credentials_file( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) @@ -607,6 +634,7 @@ def test_migration_service_client_client_options_from_dict(): quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) @@ -645,6 +673,7 @@ def test_migration_service_client_create_channel_credentials_file( quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, ) # test that the credentials from file are saved and used as the credentials. @@ -3044,6 +3073,28 @@ def test_migration_service_transport_auth_adc(transport_class): ) +@pytest.mark.parametrize( + "transport_class", + [ + transports.MigrationServiceGrpcTransport, + transports.MigrationServiceGrpcAsyncIOTransport, + ], +) +def test_migration_service_transport_auth_gdch_credentials(transport_class): + host = "https://language.com" + api_audience_tests = [None, "https://language2.com"] + api_audience_expect = [host, "https://language2.com"] + for t, e in zip(api_audience_tests, api_audience_expect): + with mock.patch.object(google.auth, "default", autospec=True) as adc: + gdch_mock = mock.MagicMock() + type(gdch_mock).with_gdch_audience = mock.PropertyMock( + return_value=gdch_mock + ) + adc.return_value = (gdch_mock, None) + transport_class(host=host, api_audience=t) + gdch_mock.with_gdch_audience.assert_called_once_with(e) + + @pytest.mark.parametrize( "transport_class,grpc_helpers", [ @@ -3543,4 +3594,5 @@ def test_api_key_credentials(client_class, transport_class): quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, always_use_jwt_access=True, + api_audience=None, )