Merge branch 'main' into functions-billing-tests

Author: busunkim96

appengine/flexible/scipy/requirements.txt

@@ -2,6 +2,6 @@ Flask==2.0.2
 gunicorn==20.1.0
 imageio==2.14.0
 numpy==1.21.4
-pillow==9.0.0
+pillow==9.0.1
 scipy==1.8.0; python_version > "3.7"
 scipy==1.7.3; python_version <= "3.7"

appengine/flexible/twilio/requirements.txt

@@ -1,3 +1,3 @@
 Flask==2.0.2
 gunicorn==20.1.0
-twilio==7.5.1
+twilio==7.6.0

appengine/standard/sendgrid/requirements.txt

@@ -1 +1 @@
-sendgrid==6.9.5
+sendgrid==6.9.6

composer/README.md

@@ -0,0 +1,12 @@
+# Cloud Composer Samples
+
+All code in this directory is samples or tooling related to [Cloud Composer](https://cloud.google.com/composer). 
+
+* [`airflow_1_samples`](/composer/airflow_1_samples) - contains DAGs compatible with Airflow 1 Composer environments. More usage examples can be found in the [Composer documentation](https://cloud.google.com/composer/docs) and [standalone samples](https://cloud.google.com/composer/docs/samples)
+* [`blog`](/composer/blog) - contains sample code used in blog posts pertaining to Cloud Composer
+* [`cicd_sample`](/composer/cicd_sample) - contains a [sample for using Cloud Build to test and sync DAGs](https://cloud.google.com/composer/docs/dag-cicd-integration-guide)
+* [`dag_test_utils`](/composer/dag_test_utils) - is a package used internally by repo maintainers
+* [`functions`](/composer/functions) - contains samples that use Cloud Functions to trigger DAGs
+* [`rest`](/composer/rest) - contains samples used in the documentation that interact with the Cloud Composer REST API
+* [`tools`](/composer/tools) - contains utilities for Cloud Composer users to manage their environments. Each utility has a corresponding README.
+* [`workflows`](/composer/workflows) - contains DAGs compatible with Airflow 2 Composer environments. More usage examples can be found in the [Composer documentation](https://cloud.google.com/composer/docs) and [standalone samples](https://cloud.google.com/composer/docs/samples)

composer/airflow_1_samples/dataflowtemplateoperator_tutorial.py

@@ -23,7 +23,6 @@
 * project_id - Google Cloud Project ID to use for the Cloud Dataflow cluster.
 * gce_zone - Google Compute Engine zone where Cloud Dataflow cluster should be
   created.
-* gce_region - Google Compute Engine region where Cloud Dataflow cluster should be
   created.
 Learn more about the difference between the two here:
 https://cloud.google.com/compute/docs/regions-zones
@@ -40,16 +39,13 @@
 bucket_path = models.Variable.get("bucket_path")
 project_id = models.Variable.get("project_id")
 gce_zone = models.Variable.get("gce_zone")
-gce_region = models.Variable.get("gce_region")
 
 
 default_args = {
     # Tell airflow to start one day ago, so that it runs as soon as you upload it
     "start_date": days_ago(1),
     "dataflow_default_options": {
         "project": project_id,
-        # Set to your region
-        "region": gce_region,
         # Set to your zone
         "zone": gce_zone,
         # This is a subfolder for storing temporary files, like the staged pipeline job.

composer/airflow_1_samples/dataflowtemplateoperator_tutorial_test.py

@@ -23,12 +23,10 @@ def set_variables(airflow_database):
     models.Variable.set("bucket_path", "gs://example_bucket")
     models.Variable.set("project_id", "example-project")
     models.Variable.set("gce_zone", "us-central1-f")
-    models.Variable.set("gce_region", "us-central1-f")
     yield
     models.Variable.delete('bucket_path')
     models.Variable.delete('project_id')
     models.Variable.delete('gce_zone')
-    models.Variable.delete('gce_region')
 
 
 def test_dag_import():

composer/cicd_sample/utils/requirements-test.txt

@@ -1,4 +1,4 @@
 pytest==6.2.3
 requests==2.27.1
 google-api-core==2.5.0
-google-resumable-media==2.1.0
+google-resumable-media==2.2.1

composer/tools/README.md

@@ -1,3 +1,4 @@
 # Composer Tools
 
-[Composer DB Migration for Airflow 1.10.14/15 -> Airflow 2](composer_db_transfer.md)
+* [Composer DB Migration for Airflow 1.10.14/15 -> Airflow 2](composer_db_transfer.md)
+* [Composer DAGs Pausing/Unpausing script](composer_dags.md)